docs.waku.org/assets/js/f06f9251.7c837a65.js

"use strict";(self.webpackChunkwaku_guide=self.webpackChunkwaku_guide||[]).push([[4930],{28453:(e,s,n)=>{n.d(s,{R:()=>a,x:()=>d});var r=n(96540);const t={},i=r.createContext(t);function a(e){const s=r.useContext(i);return r.useMemo(function(){return"function"==typeof e?e(s):{...s,...e}},[s,e])}function d(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:a(e.components),r.createElement(i.Provider,{value:s},e.children)}},96278:(e,s,n)=>{n.r(s),n.d(s,{assets:()=>o,contentTitle:()=>d,default:()=>h,frontMatter:()=>a,metadata:()=>r,toc:()=>l});const r=JSON.parse('{"id":"learn/research/benchmarks/test-results-summary","title":"Performance Benchmarks and Test Reports","description":"Introduction","source":"@site/docs/learn/research/benchmarks/test-results-summary.md","sourceDirName":"learn/research/benchmarks","slug":"/learn/research/benchmarks/test-results-summary","permalink":"/learn/research/benchmarks/test-results-summary","draft":false,"unlisted":false,"editUrl":"https://github.com/waku-org/docs.waku.org/tree/develop/docs/learn/research/benchmarks/test-results-summary.md","tags":[],"version":"current","lastUpdatedAt":null,"frontMatter":{"title":"Performance Benchmarks and Test Reports"},"sidebar":"learn","previous":{"title":"PostgreSQL","permalink":"/learn/research/benchmarks/postgres-adoption"},"next":{"title":"Capped Bandwidth in Waku","permalink":"/learn/research/research-and-studies/capped-bandwidth"}}');var t=n(74848),i=n(28453);const a={title:"Performance Benchmarks and Test Reports"},d=void 0,o={},l=[{value:"Introduction",id:"introduction",level:2},{value:"TL;DR",id:"tldr",level:2},{value:"Insights",id:"insights",level:2},{value:"Relay Bandwidth Usage: nwaku v0.34.0",id:"relay-bandwidth-usage-nwaku-v0340",level:3},{value:"Message Propagation Latency: nwaku v0.34.0-rc1",id:"message-propagation-latency-nwaku-v0340-rc1",level:3},{value:"Discv5 Bandwidth Usage: nwaku v0.34.0",id:"discv5-bandwidth-usage-nwaku-v0340",level:3},{value:"Testing",id:"testing",level:2},{value:"DST",id:"dst",level:3},{value:"QA",id:"qa",level:3},{value:"nwaku",id:"nwaku",level:3},{value:"Research",id:"research",level:3}];function c(e){const s={a:"a",blockquote:"blockquote",br:"br",code:"code",h2:"h2",h3:"h3",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,i.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(s.h2,{id:"introduction",children:"Introduction"}),"\n",(0,t.jsx)(s.p,{children:"This page summarises key performance metrics for nwaku and provides links to detailed test reports."}),"\n",(0,t.jsxs)(s.blockquote,{children:["\n",(0,t.jsx)(s.h2,{id:"tldr",children:"TL;DR"}),"\n",(0,t.jsxs)(s.ul,{children:["\n",(0,t.jsxs)(s.li,{children:["Average Waku bandwidth usage: ~",(0,t.jsx)(s.strong,{children:"10 KB/s"})," (minus discv5 Discovery) for 1KB message size and message injection rate of 1msg/s.",(0,t.jsx)(s.br,{}),"\n","Confirmed for topologies of up to 2000 Relay nodes."]}),"\n",(0,t.jsxs)(s.li,{children:["Average time for a message to propagate to 100% of nodes: ",(0,t.jsx)(s.strong,{children:"0.4s"})," for topologies of up to 2000 Relay nodes."]}),"\n",(0,t.jsxs)(s.li,{children:["Average per-node bandwidth usage of the discv5 protocol: ",(0,t.jsx)(s.strong,{children:"8 KB/s"})," for incoming traffic and ",(0,t.jsx)(s.strong,{children:"7.4 KB/s"})," for outgoing traffic,",(0,t.jsx)(s.br,{}),"\n","in a network with 100 continuously online nodes."]}),"\n",(0,t.jsxs)(s.li,{children:["Future improvements: A messaging API is currently in development to streamline interactions with the Waku protocol suite.",(0,t.jsx)(s.br,{}),"\n","Once completed, it will enable benchmarking at the messaging API level, allowing applications to more easily compare their",(0,t.jsx)(s.br,{}),"\n","own performance results."]}),"\n"]}),"\n"]}),"\n",(0,t.jsx)(s.h2,{id:"insights",children:"Insights"}),"\n",(0,t.jsx)(s.h3,{id:"relay-bandwidth-usage-nwaku-v0340",children:"Relay Bandwidth Usage: nwaku v0.34.0"}),"\n",(0,t.jsxs)(s.p,{children:["The average per-node ",(0,t.jsx)(s.code,{children:"libp2p"})," bandwidth usage in a 1000-node Relay network with 1KB messages at varying injection rates."]}),"\n",(0,t.jsxs)(s.table,{children:[(0,t.jsx)(s.thead,{children:(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.th,{children:"Message Injection Rate"}),(0,t.jsx)(s.th,{children:"Average libp2p incoming bandwidth (KB/s)"}),(0,t.jsx)(s.th,{children:"Average libp2p outgoing bandwidth (KB/s)"})]})}),(0,t.jsxs)(s.tbody,{children:[(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"1 msg/s"}),(0,t.jsx)(s.td,{children:"~10.1"}),(0,t.jsx)(s.td,{children:"~10.3"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"1 msg/10s"}),(0,t.jsx)(s.td,{children:"~1.8"}),(0,t.jsx)(s.td,{children:"~1.9"})]})]})]}),"\n",(0,t.jsx)(s.h3,{id:"message-propagation-latency-nwaku-v0340-rc1",children:"Message Propagation Latency: nwaku v0.34.0-rc1"}),"\n",(0,t.jsxs)(s.p,{children:["The message propagation latency is measured as the total time for a message to reach all nodes.",(0,t.jsx)(s.br,{}),"\n","We compare the latency in different network configurations for the following simulation parameters:"]}),"\n",(0,t.jsxs)(s.ul,{children:["\n",(0,t.jsx)(s.li,{children:"Total messages published: 600"}),"\n",(0,t.jsx)(s.li,{children:"Message size: 1KB"}),"\n",(0,t.jsx)(s.li,{children:"Message injection rate: 1msg/s"}),"\n"]}),"\n",(0,t.jsx)(s.p,{children:"The different network configurations tested are:"}),"\n",(0,t.jsxs)(s.ul,{children:["\n",(0,t.jsx)(s.li,{children:"Relay Config: 1000 nodes with relay enabled"}),"\n",(0,t.jsx)(s.li,{children:"Mixed Config: 210 nodes, consisting of bootstrap nodes, filter clients and servers, lightpush clients and servers, store nodes"}),"\n",(0,t.jsx)(s.li,{children:"Non-persistent Relay Config: 500 persistent relay nodes, 10 store nodes and 100 non-persistent relay nodes"}),"\n"]}),"\n",(0,t.jsx)(s.p,{children:"Click on a specific config to see the detailed test report."}),"\n",(0,t.jsxs)(s.table,{children:[(0,t.jsx)(s.thead,{children:(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.th,{children:"Config"}),(0,t.jsx)(s.th,{children:"Average Message Propagation Latency (s)"}),(0,t.jsx)(s.th,{children:"Max Message Propagation Latency (s)"})]})}),(0,t.jsxs)(s.tbody,{children:[(0,t.jsxs)(s.tr,{children:[(0,t.jsxs)(s.td,{children:[(0,t.jsx)(s.a,{href:"https://www.notion.so/Waku-regression-testing-v0-34-1618f96fb65c803bb7bad6ecd6bafff9",children:"Relay"})," (1000 nodes)"]}),(0,t.jsx)(s.td,{children:"0.05"}),(0,t.jsx)(s.td,{children:"1.6"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsxs)(s.td,{children:[(0,t.jsx)(s.a,{href:"https://www.notion.so/Mixed-environment-analysis-1688f96fb65c809eb235c59b97d6e15b",children:"Mixed"})," (210 nodes)"]}),(0,t.jsx)(s.td,{children:"0.0125"}),(0,t.jsx)(s.td,{children:"0.007"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsxs)(s.td,{children:[(0,t.jsx)(s.a,{href:"https://www.notion.so/High-Churn-Relay-Store-Reliability-16c8f96fb65c8008bacaf5e86881160c",children:"Non-persistent Relay"})," (510 nodes)"]}),(0,t.jsx)(s.td,{children:"0.0125"}),(0,t.jsx)(s.td,{children:"0.25"})]})]})]}),"\n",(0,t.jsx)(s.h3,{id:"discv5-bandwidth-usage-nwaku-v0340",children:"Discv5 Bandwidth Usage: nwaku v0.34.0"}),"\n",(0,t.jsxs)(s.p,{children:["The average bandwidth usage of discv5 for a network of 100 nodes and message injection rate of 0 or 1msg/s.",(0,t.jsx)(s.br,{}),"\n","The measurements are based on a stable network where all nodes have already connected to peers to form a healthy mesh."]}),"\n",(0,t.jsxs)(s.table,{children:[(0,t.jsx)(s.thead,{children:(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.th,{children:"Message size"}),(0,t.jsx)(s.th,{children:"Average discv5 incoming bandwidth (KB/s)"}),(0,t.jsx)(s.th,{children:"Average discv5 outgoing bandwidth (KB/s)"})]})}),(0,t.jsxs)(s.tbody,{children:[(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"no message injection"}),(0,t.jsx)(s.td,{children:"7.88"}),(0,t.jsx)(s.td,{children:"6.70"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"1KB"}),(0,t.jsx)(s.td,{children:"8.04"}),(0,t.jsx)(s.td,{children:"7.40"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"10KB"}),(0,t.jsx)(s.td,{children:"8.03"}),(0,t.jsx)(s.td,{children:"7.45"})]})]})]}),"\n",(0,t.jsx)(s.h2,{id:"testing",children:"Testing"}),"\n",(0,t.jsx)(s.h3,{id:"dst",children:"DST"}),"\n",(0,t.jsxs)(s.p,{children:["The VAC DST team performs regression testing on all new ",(0,t.jsx)(s.strong,{children:"nwaku"})," releases, comparing performance with previous versions.",(0,t.jsx)(s.br,{}),"\n","They simulate large Waku networks with a variety of network and protocol configurations that are representative of real-world usage."]}),"\n",(0,t.jsxs)(s.p,{children:[(0,t.jsx)(s.strong,{children:"Test Reports"}),": ",(0,t.jsx)(s.a,{href:"https://www.notion.so/DST-Reports-1228f96fb65c80729cd1d98a7496fe6f",children:"DST Reports"})]}),"\n",(0,t.jsx)(s.h3,{id:"qa",children:"QA"}),"\n",(0,t.jsxs)(s.p,{children:["The VAC QA team performs interoperability tests for ",(0,t.jsx)(s.strong,{children:"nwaku"})," and ",(0,t.jsx)(s.strong,{children:"go-waku"})," using the latest main branch builds.",(0,t.jsx)(s.br,{}),"\n","These tests run daily and verify protocol functionality by targeting specific features of each protocol."]}),"\n",(0,t.jsxs)(s.p,{children:[(0,t.jsx)(s.strong,{children:"Test Reports"}),": ",(0,t.jsx)(s.a,{href:"https://discord.com/channels/1110799176264056863/1196933819614363678",children:"QA Reports"})]}),"\n",(0,t.jsx)(s.h3,{id:"nwaku",children:"nwaku"}),"\n",(0,t.jsxs)(s.p,{children:["The ",(0,t.jsx)(s.strong,{children:"nwaku"})," team follows a structured release procedure for all release candidates.",(0,t.jsx)(s.br,{}),"\n","This involves deploying RCs to ",(0,t.jsx)(s.code,{children:"status.staging"})," fleet for validation and performing sanity checks."]}),"\n",(0,t.jsxs)(s.p,{children:[(0,t.jsx)(s.strong,{children:"Release Process"}),": ",(0,t.jsx)(s.a,{href:"https://github.com/waku-org/nwaku/blob/master/.github/ISSUE_TEMPLATE/prepare_release.md",children:"nwaku Release Procedure"})]}),"\n",(0,t.jsx)(s.h3,{id:"research",children:"Research"}),"\n",(0,t.jsxs)(s.p,{children:["The Waku Research team conducts a variety of benchmarking, performance testing, proof-of-concept validations and debugging efforts.",(0,t.jsx)(s.br,{}),"\n","They also maintain a Waku simulator designed for small-scale, single-purpose, on-demand testing."]}),"\n",(0,t.jsxs)(s.p,{children:[(0,t.jsx)(s.strong,{children:"Test Reports"}),": ",(0,t.jsx)(s.a,{href:"https://www.notion.so/Miscellaneous-2c02516248db4a28ba8cb2797a40d1bb",children:"Waku Research Reports"})]}),"\n",(0,t.jsxs)(s.p,{children:[(0,t.jsx)(s.strong,{children:"Waku Simulator"}),": ",(0,t.jsx)(s.a,{href:"https://waku-org.github.io/waku-simulator/",children:"Waku Simulator Book"})]})]})}function h(e={}){const{wrapper:s}={...(0,i.R)(),...e.components};return s?(0,t.jsx)(s,{...e,children:(0,t.jsx)(c,{...e})}):c(e)}}}]);