From 49974402254a793a7ea1865515813c5ec94e0cc5 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Thu, 5 Jun 2025 11:19:00 +0300 Subject: [PATCH] chore: add reliability tests package with longevity tests (#2361) * chore: add reliability tests package with longevity tests * chore: add reliability tests package with longevity tests * chore: fix ci errors * chore: fix * chore: fix timeout --- .github/workflows/test-reliability.yml | 37 ++++ package-lock.json | 55 ++++++ package.json | 2 + packages/reliability-tests/.eslintrc.cjs | 12 ++ packages/reliability-tests/.mocharc.cjs | 13 ++ packages/reliability-tests/README.md | 26 +++ packages/reliability-tests/package.json | 92 ++++++++++ packages/reliability-tests/src/run-tests.js | 50 ++++++ .../reliability-tests/tests/longevity.spec.ts | 164 ++++++++++++++++++ packages/reliability-tests/tsconfig.dev.json | 3 + packages/reliability-tests/tsconfig.json | 10 ++ 11 files changed, 464 insertions(+) create mode 100644 .github/workflows/test-reliability.yml create mode 100644 packages/reliability-tests/.eslintrc.cjs create mode 100644 packages/reliability-tests/.mocharc.cjs create mode 100644 packages/reliability-tests/README.md create mode 100644 packages/reliability-tests/package.json create mode 100644 packages/reliability-tests/src/run-tests.js create mode 100644 packages/reliability-tests/tests/longevity.spec.ts create mode 100644 packages/reliability-tests/tsconfig.dev.json create mode 100644 packages/reliability-tests/tsconfig.json diff --git a/.github/workflows/test-reliability.yml b/.github/workflows/test-reliability.yml new file mode 100644 index 0000000000..6c9a42fc7e --- /dev/null +++ b/.github/workflows/test-reliability.yml @@ -0,0 +1,37 @@ +name: Run Reliability Test + +on: + workflow_dispatch: + push: + branches: + - "chore/longevity-tests" + +env: + NODE_JS: "20" + +jobs: + node: + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + steps: + - uses: actions/checkout@v3 + with: + repository: waku-org/js-waku + + - name: Remove unwanted software + uses: ./.github/actions/prune-vm + + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_JS }} + + - uses: ./.github/actions/npm + + - run: npm run build:esm + + - name: Run tests + timeout-minutes: 150 + run: npm run test:longevity diff --git a/package-lock.json b/package-lock.json index f7ddadd12a..80694b7788 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "packages/sds", "packages/rln", "packages/tests", + "packages/reliability-tests", "packages/headless-tests", "packages/browser-tests", "packages/build-utils", @@ -11751,6 +11752,10 @@ "resolved": "packages/relay", "link": true }, + "node_modules/@waku/reliability-tests": { + "resolved": "packages/reliability-tests", + "link": true + }, "node_modules/@waku/rln": { "resolved": "packages/rln", "link": true @@ -44165,6 +44170,56 @@ } } }, + "packages/reliability-tests": { + "name": "@waku/reliability-tests", + "version": "0.0.1", + "license": "MIT OR Apache-2.0", + "dependencies": { + "@libp2p/interface-compliance-tests": "^6.0.1", + "@libp2p/peer-id": "^5.0.1", + "@waku/core": "*", + "@waku/enr": "*", + "@waku/interfaces": "*", + "@waku/utils": "*", + "app-root-path": "^3.1.0", + "chai-as-promised": "^7.1.1", + "debug": "^4.3.4", + "dockerode": "^4.0.2", + "fast-check": "^3.19.0", + "p-retry": "^6.1.0", + "p-timeout": "^6.1.0", + "portfinder": "^1.0.32", + "sinon": "^18.0.0", + "tail": "^2.2.6" + }, + "devDependencies": { + "@libp2p/bootstrap": "^11.0.1", + "@libp2p/crypto": "^5.0.1", + "@types/chai": "^4.3.11", + "@types/dockerode": "^3.3.19", + "@types/mocha": "^10.0.6", + "@types/sinon": "^17.0.3", + "@types/tail": "^2.2.3", + "@waku/discovery": "*", + "@waku/message-encryption": "*", + "@waku/relay": "*", + "@waku/sdk": "*", + "allure-commandline": "^2.27.0", + "allure-mocha": "^2.9.2", + "chai": "^4.3.10", + "cspell": "^8.6.1", + "datastore-core": "^10.0.2", + "debug": "^4.3.4", + "interface-datastore": "^8.2.10", + "libp2p": "2.1.8", + "mocha": "^10.3.0", + "mocha-multi-reporters": "^1.5.1", + "npm-run-all": "^4.1.5" + }, + "engines": { + "node": ">=20" + } + }, "packages/rln": { "name": "@waku/rln", "version": "0.1.5", diff --git a/package.json b/package.json index 3aee7811ca..e5f61ed7bf 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "packages/sds", "packages/rln", "packages/tests", + "packages/reliability-tests", "packages/headless-tests", "packages/browser-tests", "packages/build-utils", @@ -33,6 +34,7 @@ "test": "NODE_ENV=test npm run test --workspaces --if-present", "test:browser": "NODE_ENV=test npm run test:browser --workspaces --if-present", "test:node": "NODE_ENV=test npm run test:node --workspaces --if-present", + "test:longevity": "npm --prefix packages/reliability-tests run test:longevity", "proto": "npm run proto --workspaces --if-present", "deploy": "node ci/deploy.js", "doc": "run-s doc:*", diff --git a/packages/reliability-tests/.eslintrc.cjs b/packages/reliability-tests/.eslintrc.cjs new file mode 100644 index 0000000000..c45b8541a5 --- /dev/null +++ b/packages/reliability-tests/.eslintrc.cjs @@ -0,0 +1,12 @@ +module.exports = { + parserOptions: { + tsconfigRootDir: __dirname, + project: "./tsconfig.dev.json" + }, + rules: { + "@typescript-eslint/no-non-null-assertion": "off" + }, + globals: { + process: true + } +}; diff --git a/packages/reliability-tests/.mocharc.cjs b/packages/reliability-tests/.mocharc.cjs new file mode 100644 index 0000000000..908101e4d5 --- /dev/null +++ b/packages/reliability-tests/.mocharc.cjs @@ -0,0 +1,13 @@ +const config = { + extension: ['ts'], + require: ['ts-node/register', 'isomorphic-fetch'], + loader: 'ts-node/esm', + 'node-option': [ + 'experimental-specifier-resolution=node', + 'loader=ts-node/esm' + ], + exit: true, + retries: 0 +}; + +module.exports = config; \ No newline at end of file diff --git a/packages/reliability-tests/README.md b/packages/reliability-tests/README.md new file mode 100644 index 0000000000..7c51e01c05 --- /dev/null +++ b/packages/reliability-tests/README.md @@ -0,0 +1,26 @@ +# Reliability Tests + +This package contains reliability and stability tests for the [js-waku](https://github.com/waku-org/js-waku) project. + +These tests are designed to run realistic message scenarios in loops over extended periods, helping identify edge cases, memory leaks, network inconsistencies, or message delivery issues that may appear over time. + +## 📄 Current Tests + +### `longevity.spec.ts` + +This is the first test in the suite. It runs a js-waku<->nwaku filter scenario in a loop for 2 hours, sending and receiving messages continuously. + +The test records: +- Message ID +- Timestamp +- Send/receive status +- Any errors during transmission + +At the end, a summary report is printed and any failures cause the test to fail. + +## 🚀 How to Run + +From the **project root**: + +```bash +npm run test:longevity diff --git a/packages/reliability-tests/package.json b/packages/reliability-tests/package.json new file mode 100644 index 0000000000..56d6f2d440 --- /dev/null +++ b/packages/reliability-tests/package.json @@ -0,0 +1,92 @@ +{ + "name": "@waku/reliability-tests", + "private": true, + "version": "0.0.1", + "description": "Waku reliability tests", + "types": "./dist/index.d.ts", + "module": "./dist/index.js", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "type": "module", + "author": "Waku Team", + "homepage": "https://github.com/waku-org/js-waku/tree/master/packages/reliability-tests#readme", + "repository": { + "type": "git", + "url": "https://github.com/waku-org/js-waku.git" + }, + "bugs": { + "url": "https://github.com/waku-org/js-waku/issues" + }, + "license": "MIT OR Apache-2.0", + "keywords": [ + "waku", + "decentralized", + "secure", + "communication", + "web3", + "ethereum", + "dapps", + "privacy" + ], + "scripts": { + "build": "run-s build:**", + "build:esm": "tsc", + "fix": "run-s fix:*", + "fix:lint": "eslint src tests --fix", + "check": "run-s check:*", + "check:lint": "eslint src tests", + "check:spelling": "cspell \"{README.md,{tests,src}/**/*.ts}\"", + "check:tsc": "tsc -p tsconfig.dev.json", + "test:longevity": "NODE_ENV=test node ./src/run-tests.js tests/longevity.spec.ts", + "reset-hard": "git clean -dfx -e .idea && git reset --hard && npm i && npm run build" + }, + "engines": { + "node": ">=20" + }, + "dependencies": { + "@libp2p/interface-compliance-tests": "^6.0.1", + "@libp2p/peer-id": "^5.0.1", + "@waku/core": "*", + "@waku/enr": "*", + "@waku/interfaces": "*", + "@waku/utils": "*", + "app-root-path": "^3.1.0", + "chai-as-promised": "^7.1.1", + "debug": "^4.3.4", + "dockerode": "^4.0.2", + "fast-check": "^3.19.0", + "p-retry": "^6.1.0", + "p-timeout": "^6.1.0", + "portfinder": "^1.0.32", + "sinon": "^18.0.0", + "tail": "^2.2.6" + }, + "devDependencies": { + "@libp2p/bootstrap": "^11.0.1", + "@libp2p/crypto": "^5.0.1", + "@types/chai": "^4.3.11", + "@types/dockerode": "^3.3.19", + "@types/mocha": "^10.0.6", + "@types/sinon": "^17.0.3", + "@types/tail": "^2.2.3", + "@waku/discovery": "*", + "@waku/message-encryption": "*", + "@waku/relay": "*", + "@waku/sdk": "*", + "allure-commandline": "^2.27.0", + "allure-mocha": "^2.9.2", + "chai": "^4.3.10", + "cspell": "^8.6.1", + "datastore-core": "^10.0.2", + "debug": "^4.3.4", + "interface-datastore": "^8.2.10", + "libp2p": "2.1.8", + "mocha": "^10.3.0", + "mocha-multi-reporters": "^1.5.1", + "npm-run-all": "^4.1.5" + } +} diff --git a/packages/reliability-tests/src/run-tests.js b/packages/reliability-tests/src/run-tests.js new file mode 100644 index 0000000000..7de7da2253 --- /dev/null +++ b/packages/reliability-tests/src/run-tests.js @@ -0,0 +1,50 @@ +import { exec, spawn } from "child_process"; +import { promisify } from "util"; + +const execAsync = promisify(exec); + +const WAKUNODE_IMAGE = process.env.WAKUNODE_IMAGE || "wakuorg/nwaku:v0.35.1"; + +async function main() { + try { + await execAsync(`docker inspect ${WAKUNODE_IMAGE}`); + console.log(`Using local image ${WAKUNODE_IMAGE}`); + } catch (error) { + console.log(`Pulling image ${WAKUNODE_IMAGE}`); + await execAsync(`docker pull ${WAKUNODE_IMAGE}`); + console.log("Image pulled"); + } + + const mochaArgs = [ + "mocha", + "--require", + "ts-node/register", + "--project", + "./tsconfig.dev.json", + ...process.argv.slice(2) + ]; + + // Run mocha tests + const mocha = spawn("npx", mochaArgs, { + stdio: "inherit", + env: { + ...process.env, + NODE_ENV: "test" + } + }); + + mocha.on("error", (error) => { + console.log(`Error running mocha tests: ${error.message}`); + process.exit(1); + }); + + mocha.on("exit", (code) => { + console.log(`Mocha tests exited with code ${code}`); + process.exit(code || 0); + }); +} + +main().catch((error) => { + console.log(error); + process.exit(1); +}); diff --git a/packages/reliability-tests/tests/longevity.spec.ts b/packages/reliability-tests/tests/longevity.spec.ts new file mode 100644 index 0000000000..ea7f9a5a96 --- /dev/null +++ b/packages/reliability-tests/tests/longevity.spec.ts @@ -0,0 +1,164 @@ +import { LightNode, Protocols } from "@waku/interfaces"; +import { + createDecoder, + createEncoder, + createLightNode, + utf8ToBytes +} from "@waku/sdk"; +import { + delay, + shardInfoToPubsubTopics, + singleShardInfosToShardInfo, + singleShardInfoToPubsubTopic +} from "@waku/utils"; +import { expect } from "chai"; + +import { + afterEachCustom, + beforeEachCustom, + makeLogFileName, + MessageCollector, + ServiceNode, + tearDownNodes +} from "../../tests/src/index.js"; + +const ContentTopic = "/waku/2/content/test.js"; + +describe("Longevity", function () { + const testDurationMs = 2 * 60 * 60 * 1000; // 2 hours + this.timeout(testDurationMs + 5 * 60 * 1000); + let waku: LightNode; + let nwaku: ServiceNode; + let messageCollector: MessageCollector; + + beforeEachCustom(this, async () => { + nwaku = new ServiceNode(makeLogFileName(this.ctx)); + messageCollector = new MessageCollector(nwaku); + }); + + afterEachCustom(this, async () => { + await tearDownNodes(nwaku, waku); + }); + + it("Filter - 2 hours", async function () { + const singleShardInfo = { clusterId: 0, shard: 0 }; + const shardInfo = singleShardInfosToShardInfo([singleShardInfo]); + + const testStart = new Date(); + + const testEnd = Date.now() + testDurationMs; + + const report: { + messageId: number; + timestamp: string; + sent: boolean; + received: boolean; + error?: string; + }[] = []; + + await nwaku.start( + { + store: true, + filter: true, + relay: true, + clusterId: 0, + shard: [0], + contentTopic: [ContentTopic] + }, + { retries: 3 } + ); + + await nwaku.ensureSubscriptions(shardInfoToPubsubTopics(shardInfo)); + + waku = await createLightNode({ networkConfig: shardInfo }); + await waku.start(); + await waku.dial(await nwaku.getMultiaddrWithId()); + await waku.waitForPeers([Protocols.Filter]); + + const decoder = createDecoder(ContentTopic, singleShardInfo); + const { error } = await waku.filter.subscribe( + [decoder], + messageCollector.callback + ); + if (error) throw error; + + const encoder = createEncoder({ + contentTopic: ContentTopic, + pubsubTopicShardInfo: singleShardInfo + }); + + expect(encoder.pubsubTopic).to.eq( + singleShardInfoToPubsubTopic(singleShardInfo) + ); + + let messageId = 0; + + while (Date.now() < testEnd) { + const now = new Date(); + const message = `ping-${messageId}`; + let sent = false; + let received = false; + let err: string | undefined; + + try { + await nwaku.sendMessage( + ServiceNode.toMessageRpcQuery({ + contentTopic: ContentTopic, + payload: utf8ToBytes(message) + }) + ); + sent = true; + + received = await messageCollector.waitForMessages(1, { + timeoutDuration: 5000 + }); + + if (received) { + messageCollector.verifyReceivedMessage(0, { + expectedMessageText: message, + expectedContentTopic: ContentTopic, + expectedPubsubTopic: shardInfoToPubsubTopics(shardInfo)[0] + }); + } + } catch (e: any) { + err = e.message || String(e); + } + + report.push({ + messageId, + timestamp: now.toISOString(), + sent, + received, + error: err + }); + + messageId++; + messageCollector.list = []; // clearing the message collector + await delay(400); + } + + const failedMessages = report.filter( + (m) => !m.sent || !m.received || m.error + ); + + console.log("\n=== Longevity Test Summary ==="); + console.log("Start time:", testStart.toISOString()); + console.log("End time:", new Date().toISOString()); + console.log("Total messages:", report.length); + console.log("Failures:", failedMessages.length); + + if (failedMessages.length > 0) { + console.log("\n--- Failed Messages ---"); + for (const fail of failedMessages) { + console.log( + `#${fail.messageId} @ ${fail.timestamp} | sent: ${fail.sent} | received: ${fail.received} | error: ${fail.error || "N/A"}` + ); + } + } + + expect( + failedMessages.length, + `Some messages failed: ${failedMessages.length}` + ).to.eq(0); + }); +}); diff --git a/packages/reliability-tests/tsconfig.dev.json b/packages/reliability-tests/tsconfig.dev.json new file mode 100644 index 0000000000..4f7c34af3c --- /dev/null +++ b/packages/reliability-tests/tsconfig.dev.json @@ -0,0 +1,3 @@ +{ + "extends": "../../tsconfig.dev" +} diff --git a/packages/reliability-tests/tsconfig.json b/packages/reliability-tests/tsconfig.json new file mode 100644 index 0000000000..65b811ed50 --- /dev/null +++ b/packages/reliability-tests/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig", + "compilerOptions": { + "allowJs": true, + "outDir": "dist/", + "rootDir": "src", + "tsBuildInfoFile": "dist/.tsbuildinfo" + }, + "include": ["src"] +}