logos-co
/
roadmap
mirror of https://github.com/logos-co/roadmap.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
roadmap/waku/updates/2023-11-20.html

172 lines
107 KiB
HTML
Raw Normal View History

Update documentation
2023-11-21 01:11:26 +00:00
<!DOCTYPE html>
Update documentation
2023-11-21 22:41:46 +00:00
<html><head><title>2023-11-20 Waku Weekly</title><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta property="og:title" content="2023-11-20 Waku Weekly"/><meta property="og:description" content="2023-11-20 Waku weekly § Waku Network Can Support 10K Users § achieved: closed last PostgreSQL issue for Store scalability confirmed no unknown blockers from Waku’s side to continue dogfooding in conversation with Status Communities started team-internal dogfooding of a test community using static sharding started fleet ownership handover process: published guidelines/list of responsibilities - https://www."/><meta property="og:image" content="https://roadmap.logos.co/static/og-image.png"/><meta property="og:width" content="1200"/><meta property="og:height" content="675"/><link rel="icon" href="../../static/icon.png"/><meta name="description" content="2023-11-20 Waku weekly § Waku Network Can Support 10K Users § achieved: closed last PostgreSQL issue for Store scalability confirmed no unknown blockers from Waku’s side to continue dogfooding in conversation with Status Communities started team-internal dogfooding of a test community using static sharding started fleet ownership handover process: published guidelines/list of responsibilities - https://www."/><meta name="generator" content="Quartz"/><link rel="preconnect" href="https://fonts.googleapis.com"/><link rel="preconnect" href="https://fonts.gstatic.com"/><link href="../../index.css" rel="stylesheet" type="text/css" spa-preserve/><link href="https://cdn.jsdelivr.net/npm/katex@0.16.0/dist/katex.min.css" rel="stylesheet" type="text/css" spa-preserve/><link href="https://fonts.googleapis.com/css2?family=IBM Plex Mono&amp;family=sans-serif:wght@400;700&amp;family=sans-serif:ital,wght@0,400;0,600;1,400;1,600&amp;display=swap" rel="stylesheet" type="text/css" spa-preserve/><script src="../../prescript.js" type="application/javascript" spa-preserve></script><script type="application/javascript" spa-preserve>const fetchData = fetch(`../../static/contentIndex.json`).then(data => data.json())</script></head><body data-slug="waku/updates/2023-11-20"><div id="quartz-root" class="page"><div id="quartz-body"><div class="left sidebar"><h1 class="page-title "><a href="../..">Logos Collective Project Roadmaps</a></h1><div class="spacer mobile-only"></div><div class="search "><div id="search-icon"><p>Search</p><div></div><svg tabIndex="0" aria-labelledby="title desc" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19.9 19.7"><title id="title">Search</title><desc id="desc">Search</desc><g class="search-path" fill="none"><path stroke-linecap="square" d="M18.5 18.3l-5.4-5.4"></path><circle cx="8" cy="8" r="7"></circle></g></svg></div><div id="search-container"><div id="search-space"><input autocomplete="off" id="search-bar" name="search" type="text" aria-label="Search for something" placeholder="Search for something"/><div id="results-container"></div></div></div></div><div class="darkmode "><input class="toggle" id="darkmode-toggle" type="checkbox" tabIndex="-1"/><label id="toggle-label-light" for="darkmode-toggle" tabIndex="-1"><svg xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" version="1.1" id="dayIcon" x="0px" y="0px" viewBox="0 0 35 35" style="enable-background:new 0 0 35 35;" xmlSpace="preserve"><title>Light mode</title><path d="M6,17.5C6,16.672,5.328,16,4.5,16h-3C0.672,16,0,16.672,0,17.5 S0.672,19,1.5,19h3C5.328,19,6,18.328,6,17.5z M7.5,26c-0.414,0-0.789,0.168-1.061,0.439l-2,2C4.168,28.711,4,29.086,4,29.5 C4,30.328,4.671,31,5.5,31c0.414,0,0.789-0.168,1.06-0.44l2-2C8.832,28.289,9,27.914,9,27.5C9,26.672,8.329,26,7.5,26z M17.5,6 C18.329,6,19,5.328,19,4.5v-3C19,0.672,18.329,0,17.5,0S16,0.672,16,1.5v3C16,5.328,16.671,6,17.5,6z M27.5,9 c0.414,0,0.789-0.168,1.06-0.439l2-2C30.832,6.289,31,5.914,31,5.5C31,4.672,30.329,4,29.5,4c-0.414,0-0.789,0.168-1.061,0.44 l-2,2C26.168,6.711,26,7.086,26,7.5C26,8.328,26.671,9,27.5,9z M6.439,8.561C6.711,8.832,7.086,9,7.5,9C8.328,9,9,8.328,9,7.5 c0
Update documentation
2023-11-21 01:11:26 +00:00
<h2 id="waku-network-can-support-10k-users"><a href="https://github.com/waku-org/pm/issues/12" class="external">Waku Network Can Support 10K Users</a><a aria-hidden="true" tabindex="-1" href="#waku-network-can-support-10k-users" class="internal"> §</a></h2>
<ul>
<li><em>achieved:</em>
<ul>
<li>closed last PostgreSQL issue for Store scalability</li>
<li>confirmed no unknown blockers from Waku’s side to continue dogfooding in conversation with Status Communities</li>
<li>started team-internal dogfooding of a test community using static sharding</li>
<li>started fleet ownership handover process: published guidelines/list of responsibilities - <a href="https://www.notion.so/Fleet-Ownership-7532aad8896d46599abac3c274189741" class="external">https://www.notion.so/Fleet-Ownership-7532aad8896d46599abac3c274189741</a></li>
</ul>
</li>
<li><em>next:</em>
<ul>
<li>continue dogfooding of Status Desktop with Status staging fleet with test community</li>
<li>training session to conclude fleet ownership handover: <a href="https://www.notion.so/Fleet-Ownership-7532aad8896d46599abac3c274189741" class="external">https://www.notion.so/Fleet-Ownership-7532aad8896d46599abac3c274189741</a></li>
</ul>
</li>
<li><em>risks:</em>
<ul>
<li>Dependency on Vac/DST to conclude ~1k nodes simulations.</li>
<li>Implementation of static sharding in Status Communities and design decisions mostly driven by go-waku developer, with minimal input from Status dev (<a href="https://github.com/status-im/status-go/pull/4161" class="external">1</a>, <a href="https://github.com/status-im/status-go/pull/4094" class="external">2</a>, <a href="https://github.com/status-im/status-go/pull/4093" class="external">3</a>). See <a href="https://github.com/status-im/status-go/issues/4057" class="external">status-go#4057</a> for remaining work. Mitigation by on-boarding Chat SDK lead on 6 Nov to drive effort.</li>
<li>lack of confidence in simulation results: results so far exhibits various artifacts and anomalies seemingly related to tooling limitations. It is therefore difficult to draw certain conclusions re Waku scalability.</li>
<li>lack of clarity in terms of Status fleet ownership, monitoring and maintenance, which is an integral part of the solution.</li>
</ul>
</li>
</ul>
<h3 id="targeted-dogfooding-for-status-communities"><a href="https://github.com/waku-org/pm/issues/97" class="external">Targeted dogfooding for Status Communities</a><a aria-hidden="true" tabindex="-1" href="#targeted-dogfooding-for-status-communities" class="internal"> §</a></h3>
<ul>
<li><em>achieved</em>: logout / login freeze, fix request on correct pubsub topic, and add missing shard information on community invite</li>
<li><em>next</em>: dogfooding</li>
</ul>
<h2 id="waku-network-can-support-1-million-users---2023-11-30"><a href="https://github.com/waku-org/pm/issues/83" class="external">Waku Network can Support 1 Million Users</a> - 2023-11-30<a aria-hidden="true" tabindex="-1" href="#waku-network-can-support-1-million-users---2023-11-30" class="internal"> §</a></h2>
<ul>
<li><em>achieved</em>:
<ul>
<li>Closed last Postgresql issue for basic Store scalability. See 10k milestone.</li>
<li>Assisted DST in setting up initial tests with the ~1K tool. Currently still fine-tuning parameters, ensuring results are consistent, etc. for smaller configurations.</li>
</ul>
</li>
<li><em>risks</em>:
<ul>
<li>Dependency on Vac/DST to run 10k nodes simulations. Tracked under
<a href="https://roadmap.logos.co/tags/vac-updates" class="external"><code>vac:dst:eng-10ktool</code></a>.</li>
<li>Wakutorsis tool is being dropped, meaning new tooling needs to be developed for 10k nodes simulations. It is currently uncertain whether such tool can be developed.</li>
<li>Large scale simulations done by Vac/DST only covered nwaku relay. go-waku, status-go simulations are not planned short term (theoretical review of Status Communities messages is), nor are simulations including request-response protocols such as store and filter.</li>
<li>lack of real world feedback/dogfooding: the complete static sharding solution involves significant changes to the Waku protocol and tech stack. Although each element is unit tested, dogfooding may hit corner cases in the integrated solution that cannot be foreseen/recreated in lab conditions.</li>
</ul>
</li>
</ul>
<h2 id="waku-network-gen-0---2023-12-01"><a href="https://github.com/waku-org/pm/issues/50" class="external">Waku Network Gen 0</a> - 2023-12-01<a aria-hidden="true" tabindex="-1" href="#waku-network-gen-0---2023-12-01" class="internal"> §</a></h2>
Update documentation
2023-11-21 22:41:46 +00:00
<ul>
<li><em>achieved</em>:
<ul>
<li>Internal dogfooding of proto-network continues.</li>
<li>Significant progress of autosharding in js-waku: autosharding function implemented, work to integrate in protocols started.</li>
</ul>
</li>
<li><em>risks</em>:
<ul>
<li>Usage of RLN in js-waku and dependency on a (centralized?) Web3Provider remains unclear as one needs to know the merkle tree state (on chain) to generate proofs.</li>
<li>We are progressively moving a nwaku engineer to a solution engineer role we need to backfill the role.</li>
<li>js-waku team is juggling between dev ex and gen 0 with only 2 engineers (3rd one just joined) so delivery in this client is likely to lag behind other clients.</li>
<li>Uncertainty as to how RLN membership mechanism would hinder application adoption, if memberships need to be distributed or obtained by registration, if staking is necessary to prevent abuse, etc.</li>
</ul>
</li>
</ul>
Update documentation
2023-11-21 01:11:26 +00:00
<h3 id="14-sharded-peer-management-and-discovery"><a href="https://github.com/waku-org/pm/issues/67" class="external">1.4: Sharded peer management and discovery</a><a aria-hidden="true" tabindex="-1" href="#14-sharded-peer-management-and-discovery" class="internal"> §</a></h3>
<p><strong>[nwaku] <a href="https://github.com/waku-org/nwaku/issues/2183" class="external">chore: improve cluster id, shards, topics flow</a></strong></p>
<ul>
<li><em>achieved</em>: Various tests updates and fixes.</li>
<li><em>next</em>: Figure out why CI passes locally only.</li>
</ul>
<h3 id="12-autosharding-for-autoscaling"><a href="https://github.com/waku-org/pm/issues/65" class="external">1.2: Autosharding for autoscaling</a><a aria-hidden="true" tabindex="-1" href="#12-autosharding-for-autoscaling" class="internal"> §</a></h3>
<p><strong>[nwaku] <a href="https://github.com/waku-org/nwaku/issues/2201" class="external">chore: allow fetching cached messages by content or pubsub topic</a></strong></p>
<ul>
<li><em>achieved</em>: failed refactor of message cache</li>
<li><em>next</em>: a better and simpler message cache</li>
</ul>
<p><strong>[js-waku] <a href="https://github.com/waku-org/js-waku/issues/1500" class="external">feat: Autosharding API for req-resp protocols</a></strong></p>
<ul>
<li><em>achieved</em>: derive pubsub topic from content topic in encoders/decoders when autosharding is specified</li>
<li><em>next</em>: node config should specify static sharding or autosharding. implement autosharded topics in all req-resp protocols</li>
</ul>
<h2 id="support-many-platforms---2024-04-30"><a href="https://github.com/waku-org/pm/issues/42" class="external">Support Many Platforms</a> - 2024-04-30<a aria-hidden="true" tabindex="-1" href="#support-many-platforms---2024-04-30" class="internal"> §</a></h2>
<h3 id="presentation-readiness-"><a href="https://github.com/waku-org/pm/issues/95" class="external">Presentation Readiness </a><a aria-hidden="true" tabindex="-1" href="#presentation-readiness-" class="internal"> §</a></h3>
<p><strong>[js-waku] <a href="https://github.com/waku-org/js-waku/issues/1666" class="external">feat: node connection state</a></strong></p>
<ul>
<li><em>achieved</em>: modified <code>ConnectionManager</code> to track connection state. exposed through read function and new event</li>
<li><em>next</em>: resolve any remaining feedback</li>
</ul>
<h2 id="other-work">Other Work<a aria-hidden="true" tabindex="-1" href="#other-work" class="internal"> §</a></h2>
<h3 id="enhancements">Enhancements<a aria-hidden="true" tabindex="-1" href="#enhancements" class="internal"> §</a></h3>
<p><strong>[nwaku] <a href="https://github.com/waku-org/nwaku/issues/2220" class="external">chore(REST): adding to <code>/admin/v1/peers</code> response lightpush and filter v2 peer info</a></strong></p>
<ul>
<li><em>achieved</em>: implemented, tested and raised PR</li>
<li><em>next</em>: get PR reviewed, implement feedback and merge</li>
</ul>
<p><strong>[nwaku] <a href="https://github.com/waku-org/nwaku/issues/2207" class="external">chore: allow text/plain contentType for rest request’s body types</a></strong></p>
<ul>
<li><em>achieved</em>: Better support for js-waku using RestApi: Allow text/plain contentType</li>
<li><em>next</em>: Bandwidth management preparation</li>
</ul>
<p><strong>[nwaku] <a href="https://github.com/waku-org/nwaku/issues/2148" class="external">chore: decouple listen and announced addresses</a></strong></p>
<ul>
<li><em>achieved</em>: got the PR reviewed, implemented feedback and merged</li>
</ul>
<h3 id="ecosystem-development">Ecosystem Development<a aria-hidden="true" tabindex="-1" href="#ecosystem-development" class="internal"> §</a></h3>
<ul>
<li><em>achieved</em>: two talks last week. Logos + Waku at Web3Privacy meetup and RLN at DCxPrague</li>
<li><em>next</em>: afk for a week</li>
</ul></article></div><div class="right sidebar"><div class="graph "><h3>Graph View</h3><div class="graph-outer"><div id="graph-container" data-cfg="{&quot;drag&quot;:true,&quot;zoom&quot;:true,&quot;depth&quot;:1,&quot;scale&quot;:1.1,&quot;repelForce&quot;:0.5,&quot;centerForce&quot;:0.3,&quot;linkDistance&quot;:30,&quot;fontSize&quot;:0.6,&quot;opacityScale&quot;:1,&quot;showTags&quot;:true,&quot;removeTags&quot;:[]}"></div><svg version="1.1" id="global-graph-icon" xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 55 55" fill="currentColor" xmlSpace="preserve"><path d="M49,0c-3.309,0-6,2.691-6,6c0,1.035,0.263,2.009,0.726,2.86l-9.829,9.829C32.542,17.634,30.846,17,29,17
s-3.542,0.634-4.898,1.688l-7.669-7.669C16.785,10.424,17,9.74,17,9c0-2.206-1.794-4-4-4S9,6.794,9,9s1.794,4,4,4
c0.74,0,1.424-0.215,2.019-0.567l7.669,7.669C21.634,21.458,21,23.154,21,25s0.634,3.542,1.688,4.897L10.024,42.562
C8.958,41.595,7.549,41,6,41c-3.309,0-6,2.691-6,6s2.691,6,6,6s6-2.691,6-6c0-1.035-0.263-2.009-0.726-2.86l12.829-12.829
c1.106,0.86,2.44,1.436,3.898,1.619v10.16c-2.833,0.478-5,2.942-5,5.91c0,3.309,2.691,6,6,6s6-2.691,6-6c0-2.967-2.167-5.431-5-5.91
v-10.16c1.458-0.183,2.792-0.759,3.898-1.619l7.669,7.669C41.215,39.576,41,40.26,41,41c0,2.206,1.794,4,4,4s4-1.794,4-4
s-1.794-4-4-4c-0.74,0-1.424,0.215-2.019,0.567l-7.669-7.669C36.366,28.542,37,26.846,37,25s-0.634-3.542-1.688-4.897l9.665-9.665
C46.042,11.405,47.451,12,49,12c3.309,0,6-2.691,6-6S52.309,0,49,0z M11,9c0-1.103,0.897-2,2-2s2,0.897,2,2s-0.897,2-2,2
S11,10.103,11,9z M6,51c-2.206,0-4-1.794-4-4s1.794-4,4-4s4,1.794,4,4S8.206,51,6,51z M33,49c0,2.206-1.794,4-4,4s-4-1.794-4-4
s1.794-4,4-4S33,46.794,33,49z M29,31c-3.309,0-6-2.691-6-6s2.691-6,6-6s6,2.691,6,6S32.309,31,29,31z M47,41c0,1.103-0.897,2-2,2
s-2-0.897-2-2s0.897-2,2-2S47,39.897,47,41z M49,10c-2.206,0-4-1.794-4-4s1.794-4,4-4s4,1.794,4,4S51.206,10,49,10z"></path></svg></div><div id="global-graph-outer"><div id="global-graph-container" data-cfg="{&quot;drag&quot;:true,&quot;zoom&quot;:true,&quot;depth&quot;:-1,&quot;scale&quot;:0.9,&quot;repelForce&quot;:0.5,&quot;centerForce&quot;:0.3,&quot;linkDistance&quot;:30,&quot;fontSize&quot;:0.6,&quot;opacityScale&quot;:1,&quot;showTags&quot;:true,&quot;removeTags&quot;:[]}"></div></div></div><div class="toc desktop-only"><button type="button" id="toc"><h3>Table of Contents</h3><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="fold"><polyline points="6 9 12 15 18 9"></polyline></svg></button><div id="toc-content"><ul class="overflow"><li class="depth-0"><a href="#2023-11-20-waku-weekly" data-for="2023-11-20-waku-weekly">2023-11-20 Waku weekly</a></li><li class="depth-1"><a href="#waku-network-can-support-10k-users" data-for="waku-network-can-support-10k-users">Waku Network Can Support 10K Users</a></li><li class="depth-2"><a href="#targeted-dogfooding-for-status-communities" data-for="targeted-dogfooding-for-status-communities">Targeted dogfooding for Status Communities</a></li><li class="depth-1"><a href="#waku-network-can-support-1-million-users---2023-11-30" data-for="waku-network-can-support-1-million-users---2023-11-30">Waku Network can Support 1 Million Users - 2023-11-30</a></li><li class="depth-1"><a href="#waku-network-gen-0---2023-12-01" data-for="waku-network-gen-0---2023-12-01">Waku Network Gen 0 - 2023-12-01</a></li><li class="depth-2"><a href="#14-sharded-peer-management-and-discovery" data-for="14-sharded-peer-management-and-discovery">1.4: Sharded peer management and discovery</a></li><li class="depth-2"><a href="#12-autosharding-for-autoscaling" data-for="12-autosharding-for-autoscaling">1.2: Autosharding for autoscaling</a></li><li class="depth-1"><a href="#support-many-platforms---2024-04-30" data-for="support-many-platforms---2024-04-30">Support Many Platforms - 2024-04-30</a></li><li class="depth-2"><a href="#presentation-readiness-" data-for="presentation-readiness-">Presentation Readiness </a></li><li class="depth-1"><a href="#other-work" data-for="other-work">Other Work</a></li><li class="depth-2"><a href="#enhancements" data-for="enhancements">Enhancements</a></li><li class="depth-2"><a href="#ecosystem-development" data-for="ecosystem-development">Ecosystem Development</a></li></ul></div></div><div class="backlinks "><h3>Backlinks</h3><ul class="overflow"><li>No backlinks found</li></ul></div></div></div><footer class><hr/><p>Created with <a href="https://quartz.jzhao.xyz/">Quartz v4.1.0</a>, © 2023</p><ul><li><a href="https://github.com/logos-co/roadmap">GitHub</a></li><li><a href="https://discord.com/invite/logos-state">Discord Community</a></li></ul></footer></div></body><script type="application/javascript">// quartz/components/scripts/quartz/components/scripts/callout.inline.ts
function toggleCallout() {
const outerBlock = this.parentElement;
outerBlock.classList.toggle(`is-collapsed`);
const collapsed = outerBlock.classList.contains(`is-collapsed`);
const height = collapsed ? this.scrollHeight : outerBlock.scrollHeight;
outerBlock.style.maxHeight = height + `px`;
let current = outerBlock;
let parent = outerBlock.parentElement;
while (parent) {
if (!parent.classList.contains(`callout`)) {
return;
}
const collapsed2 = parent.classList.contains(`is-collapsed`);
const height2 = collapsed2 ? parent.scrollHeight : parent.scrollHeight + current.scrollHeight;
parent.style.maxHeight = height2 + `px`;
current = parent;
parent = parent.parentElement;
}
}
function setupCallout() {
const collapsible = document.getElementsByClassName(
`callout is-collapsible`
);
for (const div of collapsible) {
const title = div.firstElementChild;
if (title) {
title.removeEventListener(`click`, toggleCallout);
title.addEventListener(`click`, toggleCallout);
const collapsed = div.classList.contains(`is-collapsed`);
const height = collapsed ? title.scrollHeight : div.scrollHeight;
div.style.maxHeight = height + `px`;
}
}
}
document.addEventListener(`nav`, setupCallout);
window.addEventListener(`resize`, setupCallout);
</script><script type="module">
import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.esm.min.mjs';
const darkMode = document.documentElement.getAttribute('saved-theme') === 'dark'
mermaid.initialize({
startOnLoad: false,
securityLevel: 'loose',
theme: darkMode ? 'dark' : 'default'
});
document.addEventListener('nav', async () => {
await mermaid.run({
querySelector: '.mermaid'
})
});
</script><script src="https://cdn.jsdelivr.net/npm/katex@0.16.7/dist/contrib/copy-tex.min.js" type="application/javascript"></script><script src="../../postscript.js" type="module"></script></html>