roadmap/codex/updates/2023-08-11.html

150 lines
20 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!DOCTYPE html>
<html><head><title>2023-08-11 Codex weekly</title><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta property="og:title" content="2023-08-11 Codex weekly"/><meta property="og:description" content="Codex update August 11 § Client § Milestone: Merkelizing block data § Initial Merkle Tree implementation - 504 Work on persisting/serializing Merkle Tree is underway, PR upcoming Milestone: Block discovery and retrieval § Continued analysis of block discovery and retrieval - _KOAm8kNQamMx-lkQvw-Iw?both=#fn5 Reviewing papers on peers sampling and related topics Wormhole Peer Sampling paper Smoothcache Starting work on simulations based on the above work Milestone: Distributed Client Testing § Continuing working on log collection/analysis and monitoring Details here 41 More related issues/PRs: 20 20 Testing and debugging Condex in continuous testing environment Debugging continuous tests 44 pod labeling 39 Infra § Milestone: Kubernetes Configuration and Management § Move Dist-Tests cluster to OVH and define naming conventions Configure Ingress Controller for Kibana/Grafana Create documentation for Kubernetes management Configure Dist/Continuous-Tests Pods logs shipping Milestone: Continuous Testing and Labeling § Watch the Continuous tests demo Implement and configure Dist-Tests labeling Set up logs shipping based on labels Improve Docker workflows and add latest tag Milestone: CI/CD and Synchronization § Set up synchronization by codex-storage Configure Codex Storage and Demo CI/CD environments Marketplace § Milestone: L2 § Taiko L2 integration Done but merge is blocked by a few issues - 483 Milestone: Marketplace Sales § Lots of cleanup and refactoring Finished refactoring state machine PR link Added support for loading nodes slots during Sales module start link DAS § Milestone: DHT simulations § Implementing a DHT in Python for the DAS simulator - py-dht."/><meta property="og:image" content="https://roadmap.logos.co/static/og-image.png"/><meta property="og:width" content="1200"/><meta property="og:height" content="675"/><link rel="icon" href="../../static/icon.png"/><meta name="description" content="Codex update August 11 § Client § Milestone: Merkelizing block data § Initial Merkle Tree implementation - 504 Work on persisting/serializing Merkle Tree is underway, PR upcoming Milestone: Block discovery and retrieval § Continued analysis of block discovery and retrieval - _KOAm8kNQamMx-lkQvw-Iw?both=#fn5 Reviewing papers on peers sampling and related topics Wormhole Peer Sampling paper Smoothcache Starting work on simulations based on the above work Milestone: Distributed Client Testing § Continuing working on log collection/analysis and monitoring Details here 41 More related issues/PRs: 20 20 Testing and debugging Condex in continuous testing environment Debugging continuous tests 44 pod labeling 39 Infra § Milestone: Kubernetes Configuration and Management § Move Dist-Tests cluster to OVH and define naming conventions Configure Ingress Controller for Kibana/Grafana Create documentation for Kubernetes management Configure Dist/Continuous-Tests Pods logs shipping Milestone: Continuous Testing and Labeling § Watch the Continuous tests demo Implement and configure Dist-Tests labeling Set up logs shipping based on labels Improve Docker workflows and add latest tag Milestone: CI/CD and Synchronization § Set up synchronization by codex-storage Configure Codex Storage and Demo CI/CD environments Marketplace § Milestone: L2 § Taiko L2 integration Done but merge is blocked by a few issues - 483 Milestone: Marketplace Sales § Lots of cleanup and refactoring Finished refactoring state machine PR link Added support for loading nodes slots during Sales module start link DAS § Milestone: DHT simulations § Implementing a DHT in Python for the DAS simulator - py-dht."/><meta name="generator" content="Logos Roadmaps"/><link rel="preconnect" href="https://fonts.googleapis.com"/><link rel="preconnect" href="https://fonts.gstatic.com"/><link href="../../index.css" rel="stylesheet" type="text/css" spa-preserve/><link href="https://cdn.jsdelivr.net/npm/katex@0.16.0/dist/katex.min.css" rel="stylesheet" type="text/css" spa-preserve/><link href="https://fonts.googleapis.com/css2?family=IBM Plex Mono&amp;family=sans-serif:wght@400;700&amp;family=sans-serif:ital,wght@0,400;0,600;1,400;1,600&amp;display=swap" rel="stylesheet" type="text/css" spa-preserve/><script src="../../prescript.js" type="application/javascript" spa-preserve></script><script type="application/javascript" spa-preserve>const fetchData = fetch(`../../static/contentIndex.json`).then(data => data.json())</script></head><body data-slug="codex/updates/2023-08-11"><div id="quartz-root" class="page"><div id="quartz-body"><div class="left sidebar"><h1 class="page-title"><a href="../..">Logos Collective Project Roadmaps</a></h1><div class="spacer mobile-only"></div><div class="search"><div id="search-icon"><p>Search</p><div></div><svg tabIndex="0" aria-labelledby="title desc" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19.9 19.7"><title id="title">Search</title><desc id="desc">Search</desc><g class="search-path" fill="none"><path stroke-linecap="square" d="M18.5 18.3l-5.4-5.4"></path><circle cx="8" cy="8" r="7"></circle></g></svg></div><div id="search-container"><div id="search-space"><input autocomplete="off" id="search-bar" name="search" type="text" aria-label="Search for something" placeholder="Search for something"/><div id="results-container"></div></div></div></div><div class="darkmode"><input class="toggle" id="darkmode-toggle" type="checkbox" tabIndex="-1"/><label id="toggle-label-light" for="darkmode-toggle" tabIndex="-1"><svg xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" version="1.1" id="dayIcon" x="0px" y="0px" viewBox="0 0 35 35" style="enable-background:new 0 0 35 35;" xmlSpace="preserve"><title>Light mode</title><path d="M6,17.5C6,16.672,5.328,16,4.5,16h-3C0.672,16,0,16.672,0,17.5 S0.672,19,1.5,19h3C5.328,19,6,18.328,6,17.5z M7.5,26c-0.414,0-0.789,0.168-1.061,0.439l-2,2C4.168,28.711,4,29.086,4,29.5 C4,30.328,4.671,31,5.5,31c0.414,0,0.789-0.168,1.06-0.44l2-2C8.832,28.289,9,27.914,9,27.5C9,26.672,8.329,26,7.5,26z M17.5,6 C18.329,6,19,5.328,19,4.5v-3C19,0.672,18.329,0,17.5,0S16,0.672,16,1.5v3C16,5.328,16.671,6,17.5,6z M27.5,9 c0.414,0,0.789-0.168,1.06-0.439l2-2C30.832,6.289,31,5.914,31,5.5C31,4.672,30.329,4,29.5,4c-0.414,0-0.789,0.168-1.061,0.44 l-2,2C26.168,6.711,26,7.086,26,7.5C26,8.328,26.671,9,27.5,9z M6.439,8.561C6.711,8.832,7.086,9,7.5,9C8.328,9,9,8.328,9,7.5 c0-0.414-0.168-0.789-0.439-1.061l-2-2C6.289,4.168,5.914,4,5.5,4C4.672,4,4,4.672,4,5.5c0,0.414,0.168,0.789,0.439,1.06 L6.439,8.561z M33.5,16h-3c-0.828,0-1.5,0.672-1.5,1.5s0.672,1.5,1.5,1.5h3c0.828,0,1.5-0.672,1.5-1.5S34.328,16,33.5,16z M28.561,26.439C28.289,26.168,27.914,26,27.5,26c-0.828,0-1.5,0.672-1.5,1.5c0,0.414,0.168,0.789,0.439,1.06l2,2 C28.711,30.832,29.086,31,29.5,31c0.828,0,1.5-0.672,1.5-1.5c0-0.414-0.168-0.789-0.439-1.061L28.561,26.439z M17.5,29 c-0.829,0-1.5,0.672-1.5,1.5v3c0,0.828,0.671,1.5,1.5,1.5s1.5-0.672,1.5-1.5v-3C19,29.672,18.329,29,17.5,29z M17.5,7 C11.71,7,7,11.71,7,17.5S11.71,28,17.5,28S28,23.29,28,17.5S23.29,7,17.5,7z M17.5,25c-4.136,0-7.5-3.364-7.5-7.5 c0-4.136,3.364-7.5,7.5-7.5c4.136,0,7.5,3.364,7.5,7.5C25,21.636,21.636,25,17.5,25z"></path></svg></label><label id="toggle-label-dark" for="darkmode-toggle" tabIndex="-1"><svg xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" version="1.1" id="nightIcon" x="0px" y="0px" viewBox="0 0 100 100" style="enable-background='new 0 0 100 100'" xmlSpace="preserve"><title>Dark mode</title><path d="M96.76,66.458c-0.853-0.852-2.15-1.064-3.23-0.534c-6.063,2.991-12.858,4.571-19.655,4.571 C62.022,70.495,50.88,65.88,42.5,57.5C29.043,44.043,25.658,23.536,34.076,6.47c0.532-1.08,0.318-2.379-0.534-3.23 c-0.851-0.852-2.15-1.064-3.23-0.534c-4.918,2.427-9.375,5.619-13.246,9.491c-9.447,9.447-14.65,22.008-14.65,35.369 c0,13.36,5.203,25.921,14.65,35.368s22.008,14.65,35.368,14.65c13.361,0,25.921-5.203,35.369-14.65 c3.872-3.871,7.064-8.328,9.491-13.246C97.826,68.608,97.611,67.309,96.76,66.458z"></path></svg></label></div><div class="toc desktop-only"><button type="button" id="toc"><h3>Table of Contents</h3><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="fold"><polyline points="6 9 12 15 18 9"></polyline></svg></button><div id="toc-content"><ul class="overflow"><li class="depth-0"><a href="#codex-update-august-11" data-for="codex-update-august-11">Codex update August 11</a></li><li class="depth-1"><a href="#client" data-for="client">Client</a></li><li class="depth-2"><a href="#milestone-merkelizing-block-data" data-for="milestone-merkelizing-block-data">Milestone: Merkelizing block data</a></li><li class="depth-2"><a href="#milestone-block-discovery-and-retrieval" data-for="milestone-block-discovery-and-retrieval">Milestone: Block discovery and retrieval</a></li><li class="depth-2"><a href="#milestone-distributed-client-testing" data-for="milestone-distributed-client-testing">Milestone: Distributed Client Testing</a></li><li class="depth-1"><a href="#infra" data-for="infra">Infra</a></li><li class="depth-2"><a href="#milestone-kubernetes-configuration-and-management" data-for="milestone-kubernetes-configuration-and-management">Milestone: Kubernetes Configuration and Management</a></li><li class="depth-2"><a href="#milestone-continuous-testing-and-labeling" data-for="milestone-continuous-testing-and-labeling">Milestone: Continuous Testing and Labeling</a></li><li class="depth-2"><a href="#milestone-cicd-and-synchronization" data-for="milestone-cicd-and-synchronization">Milestone: CI/CD and Synchronization</a></li><li class="depth-1"><a href="#marketplace" data-for="marketplace">Marketplace</a></li><li class="depth-2"><a href="#milestone-l2" data-for="milestone-l2">Milestone: L2</a></li><li class="depth-2"><a href="#milestone-marketplace-sales" data-for="milestone-marketplace-sales">Milestone: Marketplace Sales</a></li><li class="depth-1"><a href="#das" data-for="das">DAS</a></li><li class="depth-2"><a href="#milestone-dht-simulations" data-for="milestone-dht-simulations">Milestone: DHT simulations</a></li></ul></div></div></div><div class="center"><div class="page-header"><div class="popover-hint"><h1 class="article-title">2023-08-11 Codex weekly</h1><p class="content-meta">Aug 17, 2023, 2 min read</p><ul class="tags"><li><a href="../../tags/codex-updates" class="internal tag-link">#codex-updates</a></li></ul></div></div><article class="popover-hint"><h1 id="codex-update-august-11">Codex update August 11<a aria-hidden="true" tabindex="-1" href="#codex-update-august-11" class="internal"> §</a></h1>
<hr/>
<h2 id="client">Client<a aria-hidden="true" tabindex="-1" href="#client" class="internal"> §</a></h2>
<h3 id="milestone-merkelizing-block-data">Milestone: Merkelizing block data<a aria-hidden="true" tabindex="-1" href="#milestone-merkelizing-block-data" class="internal"> §</a></h3>
<ul>
<li>Initial Merkle Tree implementation - <a href="https://github.com/codex-storage/nim-codex/pull/504" class="external">504</a></li>
<li>Work on persisting/serializing Merkle Tree is underway, PR upcoming</li>
</ul>
<h3 id="milestone-block-discovery-and-retrieval">Milestone: Block discovery and retrieval<a aria-hidden="true" tabindex="-1" href="#milestone-block-discovery-and-retrieval" class="internal"> §</a></h3>
<ul>
<li>Continued analysis of block discovery and retrieval - <a href="https://hackmd.io/_KOAm8kNQamMx-lkQvw-Iw?both=#fn5" class="external">_KOAm8kNQamMx-lkQvw-Iw?both=#fn5</a>
<ul>
<li>Reviewing papers on peers sampling and related topics
<ul>
<li><a href="http://publicatio.bibl.u-szeged.hu/3895/1/p2p13.pdf" class="external">Wormhole Peer Sampling paper</a></li>
<li><a href="https://dl.acm.org/doi/10.1145/2713168.2713182" class="external">Smoothcache</a></li>
</ul>
</li>
</ul>
</li>
<li>Starting work on simulations based on the above work</li>
</ul>
<h3 id="milestone-distributed-client-testing">Milestone: Distributed Client Testing<a aria-hidden="true" tabindex="-1" href="#milestone-distributed-client-testing" class="internal"> §</a></h3>
<ul>
<li>Continuing working on log collection/analysis and monitoring
<ul>
<li>Details here <a href="https://github.com/codex-storage/cs-codex-dist-tests/pull/41" class="external">41</a></li>
<li>More related issues/PRs:
<ul>
<li><a href="https://github.com/codex-storage/infra-codex/pull/20" class="external">20</a></li>
<li><a href="https://github.com/codex-storage/infra-codex/pull/20" class="external">20</a></li>
</ul>
</li>
</ul>
</li>
<li>Testing and debugging Condex in continuous testing environment
<ul>
<li>Debugging continuous tests <a href="https://github.com/codex-storage/cs-codex-dist-tests/pull/44" class="external">44</a></li>
<li>pod labeling <a href="https://github.com/codex-storage/cs-codex-dist-tests/issues/39" class="external">39</a></li>
</ul>
</li>
</ul>
<hr/>
<h2 id="infra">Infra<a aria-hidden="true" tabindex="-1" href="#infra" class="internal"> §</a></h2>
<h3 id="milestone-kubernetes-configuration-and-management">Milestone: Kubernetes Configuration and Management<a aria-hidden="true" tabindex="-1" href="#milestone-kubernetes-configuration-and-management" class="internal"> §</a></h3>
<ul>
<li>Move Dist-Tests cluster to OVH and define naming conventions</li>
<li>Configure Ingress Controller for Kibana/Grafana</li>
<li><strong>Create documentation for Kubernetes management</strong></li>
<li><strong>Configure Dist/Continuous-Tests Pods logs shipping</strong></li>
</ul>
<h3 id="milestone-continuous-testing-and-labeling">Milestone: Continuous Testing and Labeling<a aria-hidden="true" tabindex="-1" href="#milestone-continuous-testing-and-labeling" class="internal"> §</a></h3>
<ul>
<li>Watch the Continuous tests demo</li>
<li>Implement and configure Dist-Tests labeling</li>
<li>Set up logs shipping based on labels</li>
<li>Improve Docker workflows and add latest tag</li>
</ul>
<h3 id="milestone-cicd-and-synchronization">Milestone: CI/CD and Synchronization<a aria-hidden="true" tabindex="-1" href="#milestone-cicd-and-synchronization" class="internal"> §</a></h3>
<ul>
<li>Set up synchronization by codex-storage</li>
<li>Configure Codex Storage and Demo CI/CD environments</li>
</ul>
<hr/>
<h2 id="marketplace">Marketplace<a aria-hidden="true" tabindex="-1" href="#marketplace" class="internal"> §</a></h2>
<h3 id="milestone-l2">Milestone: L2<a aria-hidden="true" tabindex="-1" href="#milestone-l2" class="internal"> §</a></h3>
<ul>
<li>Taiko L2 integration
<ul>
<li>Done but merge is blocked by a few issues - <a href="https://github.com/codex-storage/nim-codex/pull/483" class="external">483</a></li>
</ul>
</li>
</ul>
<h3 id="milestone-marketplace-sales">Milestone: Marketplace Sales<a aria-hidden="true" tabindex="-1" href="#milestone-marketplace-sales" class="internal"> §</a></h3>
<ul>
<li>Lots of cleanup and refactoring
<ul>
<li>Finished refactoring state machine PR <a href="https://github.com/codex-storage/nim-codex/pull/469" class="external">link</a></li>
<li>Added support for loading nodes slots during Sales module start <a href="https://github.com/codex-storage/nim-codex/pull/510" class="external">link</a></li>
</ul>
</li>
</ul>
<hr/>
<h2 id="das">DAS<a aria-hidden="true" tabindex="-1" href="#das" class="internal"> §</a></h2>
<h3 id="milestone-dht-simulations">Milestone: DHT simulations<a aria-hidden="true" tabindex="-1" href="#milestone-dht-simulations" class="internal"> §</a></h3>
<ul>
<li>Implementing a DHT in Python for the DAS simulator - <a href="https://github.com/cortze/py-dht" class="external">py-dht</a>.</li>
</ul>
<p>NOTE: Several people are/where out during the last few weeks, so some milestones are paused until they are back</p></article></div><div class="right sidebar"><div class="graph"><h3>Graph View</h3><div class="graph-outer"><div id="graph-container" data-cfg="{&quot;drag&quot;:true,&quot;zoom&quot;:true,&quot;depth&quot;:1,&quot;scale&quot;:1.1,&quot;repelForce&quot;:0.5,&quot;centerForce&quot;:0.3,&quot;linkDistance&quot;:30,&quot;fontSize&quot;:0.6,&quot;opacityScale&quot;:1}"></div><svg version="1.1" id="global-graph-icon" xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 55 55" fill="currentColor" xmlSpace="preserve"><path d="M49,0c-3.309,0-6,2.691-6,6c0,1.035,0.263,2.009,0.726,2.86l-9.829,9.829C32.542,17.634,30.846,17,29,17
s-3.542,0.634-4.898,1.688l-7.669-7.669C16.785,10.424,17,9.74,17,9c0-2.206-1.794-4-4-4S9,6.794,9,9s1.794,4,4,4
c0.74,0,1.424-0.215,2.019-0.567l7.669,7.669C21.634,21.458,21,23.154,21,25s0.634,3.542,1.688,4.897L10.024,42.562
C8.958,41.595,7.549,41,6,41c-3.309,0-6,2.691-6,6s2.691,6,6,6s6-2.691,6-6c0-1.035-0.263-2.009-0.726-2.86l12.829-12.829
c1.106,0.86,2.44,1.436,3.898,1.619v10.16c-2.833,0.478-5,2.942-5,5.91c0,3.309,2.691,6,6,6s6-2.691,6-6c0-2.967-2.167-5.431-5-5.91
v-10.16c1.458-0.183,2.792-0.759,3.898-1.619l7.669,7.669C41.215,39.576,41,40.26,41,41c0,2.206,1.794,4,4,4s4-1.794,4-4
s-1.794-4-4-4c-0.74,0-1.424,0.215-2.019,0.567l-7.669-7.669C36.366,28.542,37,26.846,37,25s-0.634-3.542-1.688-4.897l9.665-9.665
C46.042,11.405,47.451,12,49,12c3.309,0,6-2.691,6-6S52.309,0,49,0z M11,9c0-1.103,0.897-2,2-2s2,0.897,2,2s-0.897,2-2,2
S11,10.103,11,9z M6,51c-2.206,0-4-1.794-4-4s1.794-4,4-4s4,1.794,4,4S8.206,51,6,51z M33,49c0,2.206-1.794,4-4,4s-4-1.794-4-4
s1.794-4,4-4S33,46.794,33,49z M29,31c-3.309,0-6-2.691-6-6s2.691-6,6-6s6,2.691,6,6S32.309,31,29,31z M47,41c0,1.103-0.897,2-2,2
s-2-0.897-2-2s0.897-2,2-2S47,39.897,47,41z M49,10c-2.206,0-4-1.794-4-4s1.794-4,4-4s4,1.794,4,4S51.206,10,49,10z"></path></svg></div><div id="global-graph-outer"><div id="global-graph-container" data-cfg="{&quot;drag&quot;:true,&quot;zoom&quot;:true,&quot;depth&quot;:-1,&quot;scale&quot;:0.9,&quot;repelForce&quot;:0.5,&quot;centerForce&quot;:0.3,&quot;linkDistance&quot;:30,&quot;fontSize&quot;:0.6,&quot;opacityScale&quot;:1}"></div></div></div><div class="backlinks"><h3>Backlinks</h3><ul class="overflow"><li>No backlinks found</li></ul></div></div></div><footer><hr/><p>Created by Logos with <a href="https://quartz.jzhao.xyz/">Quartz v4.0.8</a>, © 2023</p><ul><li><a href="https://github.com/logos-co/roadmap">GitHub</a></li><li><a href="https://discord.com/invite/logos-state">Discord Community</a></li></ul></footer></div></body><script type="application/javascript">// quartz/components/scripts/quartz/components/scripts/callout.inline.ts
function toggleCallout() {
const outerBlock = this.parentElement;
outerBlock.classList.toggle(`is-collapsed`);
const collapsed = outerBlock.classList.contains(`is-collapsed`);
const height = collapsed ? this.scrollHeight : outerBlock.scrollHeight;
outerBlock.style.maxHeight = height + `px`;
let current = outerBlock;
let parent = outerBlock.parentElement;
while (parent) {
if (!parent.classList.contains(`callout`)) {
return;
}
const collapsed2 = parent.classList.contains(`is-collapsed`);
const height2 = collapsed2 ? parent.scrollHeight : parent.scrollHeight + current.scrollHeight;
parent.style.maxHeight = height2 + `px`;
current = parent;
parent = parent.parentElement;
}
}
function setupCallout() {
const collapsible = document.getElementsByClassName(
`callout is-collapsible`
);
for (const div of collapsible) {
const title = div.firstElementChild;
if (title) {
title.removeEventListener(`click`, toggleCallout);
title.addEventListener(`click`, toggleCallout);
const collapsed = div.classList.contains(`is-collapsed`);
const height = collapsed ? title.scrollHeight : div.scrollHeight;
div.style.maxHeight = height + `px`;
}
}
}
document.addEventListener(`nav`, setupCallout);
window.addEventListener(`resize`, setupCallout);
</script><script type="module">
import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.esm.min.mjs';
const darkMode = document.documentElement.getAttribute('saved-theme') === 'dark'
mermaid.initialize({
startOnLoad: false,
securityLevel: 'loose',
theme: darkMode ? 'dark' : 'default'
});
document.addEventListener('nav', async () => {
await mermaid.run({
querySelector: '.mermaid'
})
});
</script><script src="https://cdn.jsdelivr.net/npm/katex@0.16.7/dist/contrib/copy-tex.min.js" type="application/javascript"></script><script src="../../postscript.js" type="module"></script></html>