logos-storage/das-dht-emulator
1
0
Fork 0
mirror of https://github.com/logos-storage/das-dht-emulator.git synced 2026-01-04 06:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial commit of the DAS DHT emulator

Browse Source 
Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
This commit is contained in:
Csaba Kiraly 2023-06-13 12:44:25 +02:00
commit ed05ebba46
No known key found for this signature in database
GPG Key ID: 0FE274EE8C95166E
26 changed files with 554 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
.gitmodules vendored Normal file
View File

@ -0,0 +1,60 @@
[submodule "vendor/nimbus-build-system"]
path = vendor/nimbus-build-system
url = https://github.com/status-im/nimbus-build-system.git
[submodule "vendor/nim-libp2p-dht"]
path = vendor/nim-libp2p-dht
url = https://github.com/codex-storage/nim-libp2p-dht.git
[submodule "vendor/nim-chronicles"]
path = vendor/nim-chronicles
url = https://github.com/status-im/nim-chronicles.git
[submodule "vendor/nim-chronos"]
path = vendor/nim-chronos
url = https://github.com/status-im/nim-chronos.git
[submodule "vendor/nim-bearssl"]
path = vendor/nim-bearssl
url = https://github.com/status-im/nim-bearssl.git
[submodule "vendor/nim-libp2p"]
path = vendor/nim-libp2p
url = https://github.com/status-im/nim-libp2p.git
[submodule "vendor/nim-faststreams"]
path = vendor/nim-faststreams
url = https://github.com/status-im/nim-faststreams.git
[submodule "vendor/nim-stew"]
path = vendor/nim-stew
url = https://github.com/status-im/nim-stew.git
[submodule "vendor/nim-json-serialization"]
path = vendor/nim-json-serialization
url = https://github.com/status-im/nim-json-serialization.git
[submodule "vendor/nim-serialization"]
path = vendor/nim-serialization
url = https://github.com/status-im/nim-serialization.git
[submodule "vendor/nimcrypto"]
path = vendor/nimcrypto
url = https://github.com/cheatfate/nimcrypto.git
[submodule "vendor/stint"]
path = vendor/stint
url = https://github.com/status-im/stint.git
[submodule "vendor/nim-secp256k1"]
path = vendor/nim-secp256k1
url = https://github.com/status-im/nim-secp256k1.git
[submodule "vendor/nim-metrics"]
path = vendor/nim-metrics
url = https://github.com/status-im/nim-metrics.git
[submodule "vendor/nim-datastore"]
path = vendor/nim-datastore
url = https://github.com/status-im/nim-datastore.git
[submodule "vendor/questionable"]
path = vendor/questionable
url = https://github.com/status-im/questionable.git
[submodule "vendor/upraises"]
path = vendor/upraises
url = https://github.com/markspanbroek/upraises.git
[submodule "vendor/nim-sqlite3-abi"]
path = vendor/nim-sqlite3-abi
url = https://github.com/arnetheduck/nim-sqlite3-abi.git
[submodule "vendor/nim-websock"]
path = vendor/nim-websock
url = https://github.com/status-im/nim-websock.git
[submodule "vendor/nim-http-utils"]
path = vendor/nim-http-utils
url = https://github.com/status-im/nim-http-utils.git

205
LICENSE-APACHEv2 Normal file
View File

@ -0,0 +1,205 @@
Copyright (c) 2022 Status Research & Development GmbH
-----------------------------------------------------
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2022 Status Research & Development GmbH
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

20
LICENSE-MIT Normal file
View File

@ -0,0 +1,20 @@
Copyright (c) 2022 Status Research & Development GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

55
Makefile Normal file
View File

@ -0,0 +1,55 @@
SHELL := bash # the shell used internally by "make"
# used inside the included makefiles
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
.PHONY: \
all \
deps \
update \
das \
clean
ifeq ($(NIM_PARAMS),)
# "variables.mk" was not included, so we update the submodules.
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
.DEFAULT:
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
$(GIT_SUBMODULE_UPDATE) && \
echo
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
#
# After restarting, it will execute its original goal, so we don't have to start a child Make here
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
else # "variables.mk" was included. Business as usual until the end of this file.
# default target, because it's the first one that doesn't start with '.'
all: | das
# must be included after the default target
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
# add a default Nim compiler argument
NIM_PARAMS += -d:release
deps: | deps-common
# Have custom deps? Add them above.
update: | update-common
# Do you need to do something extra for this target?
# building Nim programs
das: | build deps
echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) "$(NIMC)" c -o:build/$@ $(NIM_PARAMS) "$@.nim"
clean: | clean-common
rm -rf build/{das}
endif # "variables.mk" was not included

9
README.md Normal file
View File

@ -0,0 +1,9 @@
# DAS emulator
Emulate DAS DHT behavior, with a few simplifying assumptions:
- the block is populated in the DHT by the builder (node 0)
- all nodes start sampling at the same time
- 1-way latency is 50ms (configurable)
- no losses in transmission (configurable)
- scaled down numbers (nodes, blocksize, etc., all configrable)

185
das.nim Normal file
View File

@ -0,0 +1,185 @@
import
std/[random, math],
chronicles,
chronos,
libp2pdht/dht,
libp2pdht/discv5/crypto as dhtcrypto,
libp2pdht/discv5/protocol as discv5_protocol,
tests/dht/test_helper
logScope:
topics = "DAS emulator"
proc bootstrapNodes(
nodecount: int,
bootnodes: seq[SignedPeerRecord],
rng = newRng(),
delay: int = 0
) : Future[seq[(discv5_protocol.Protocol, PrivateKey)]] {.async.} =
debug "---- STARTING BOOSTRAPS ---"
for i in 0..<nodecount:
try:
let privKey = PrivateKey.example(rng)
let node = initDiscoveryNode(rng, privKey, localAddress(20302 + i), bootnodes)
await node.start()
result.add((node, privKey))
if delay > 0:
await sleepAsync(chronos.milliseconds(delay))
except TransportOsError as e:
echo "skipping node ",i ,":", e.msg
#await allFutures(result.mapIt(it.bootstrap())) # this waits for bootstrap based on bootENode, which includes bonding with all its ping pongs
proc bootstrapNetwork(
nodecount: int,
rng = newRng(),
delay: int = 0
) : Future[seq[(discv5_protocol.Protocol, PrivateKey)]] {.async.} =
let
bootNodeKey = PrivateKey.fromHex(
"a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617")
.expect("Valid private key hex")
bootNodeAddr = localAddress(20301)
bootNode = initDiscoveryNode(rng, bootNodeKey, bootNodeAddr, @[]) # just a shortcut for new and open
#waitFor bootNode.bootstrap() # immediate, since no bootnodes are defined above
var res = await bootstrapNodes(nodecount - 1,
@[bootnode.localNode.record],
rng,
delay)
res.insert((bootNode, bootNodeKey), 0)
return res
proc toNodeId(data: openArray[byte]): NodeId =
readUintBE[256](keccak256.digest(data).data)
proc segmentData(s: int, segmentsize: int) : seq[byte] =
result = newSeq[byte](segmentsize)
var
r = s
i = 0
while r > 0:
assert(i<segmentsize)
result[i] = byte(r mod 256)
r = r div 256
i+=1
proc sample(s: Slice[int], len: int): seq[int] =
# random sample without replacement
# TODO: not the best for small len
assert s.a <= s.b
var all = s.b - s.a + 1
var count = len
if len >= all div 10: # add better algo selector
var generated = newSeq[bool](all) # Initialized to false.
while count != 0:
let n = rand(s)
if not generated[n - s.a]:
generated[n - s.a] = true
result.add n
dec count
else:
while count != 0:
let n = rand(s)
if not (n in result):
result.add n
dec count
when isMainModule:
proc main() {.async.} =
let
nodecount = 100
delay_pernode = 10 # in millisec
delay_init = 15*1000 # in millisec
blocksize = 256
segmentsize = 2
samplesize = 3
upload_timeout = 5.seconds
sampling_timeout = 5.seconds
assert(log2(blocksize.float).ceil.int <= segmentsize * 8 )
assert(samplesize <= blocksize)
var
segmentIDs = newSeq[NodeId](blocksize)
# start network
let
rng = newRng()
nodes = await bootstrapNetwork(nodecount=nodecount, delay=delay_pernode)
# wait for network to settle
await sleepAsync(chronos.milliseconds(delay_init))
# generate block and push data
info "starting upload to DHT"
let startTime = Moment.now()
var futs = newSeq[Future[seq[Node]]]()
for s in 0 ..< blocksize:
let
segment = segmentData(s, segmentsize)
key = toNodeId(segment)
segmentIDs[s] = key
futs.add(nodes[0][0].addValue(key, segment))
let pass = await allFutures(futs).withTimeout(upload_timeout)
info "uploaded to DHT", by = 0, pass, time = Moment.now() - startTime
# sample
proc startSamplingDA(n: discv5_protocol.Protocol): seq[Future[DiscResult[seq[byte]]]] =
## Generate random sample and start the sampling process
var futs = newSeq[Future[DiscResult[seq[byte]]]]()
let sample = sample(0 ..< blocksize, samplesize)
debug "starting sampling", by = n, sample
for s in sample:
let fut = n.getValue(segmentIDs[s])
futs.add(fut)
return futs
proc sampleDA(n: discv5_protocol.Protocol): Future[(bool, int, Duration)] {.async.} =
## Sample and return detailed results of sampling
let startTime = Moment.now()
var futs = startSamplingDA(n)
# test is passed if all segments are retrieved in time
let pass = await allFutures(futs).withTimeout(sampling_timeout)
var passcount: int
for f in futs:
if f.finished():
passcount += 1
let time = Moment.now() - startTime
info "sample", by = n.localNode, pass, cnt = passcount, time
return (pass, passcount, time)
# all nodes start sampling in parallel
var samplings = newSeq[Future[(bool, int, Duration)]]()
for n in 1 ..< nodecount:
samplings.add(sampleDA(nodes[n][0]))
await allFutures(samplings)
# print statistics
var
passed = 0
for f in samplings:
if f.finished():
let (pass, passcount, time) = await f
passed += pass.int
debug "sampleStats", pass, cnt = passcount, time
else:
error "This should not happen!"
info "sampleStats", passed, total = samplings.len, ratio = passed/samplings.len
waitfor main()
# proc teardownAll() =
# for (n, _) in nodes: # if last test is enabled, we need nodes[1..^1] here
# await n.closeWait()

1
vendor/nim-bearssl vendored Submodule

@ -0,0 +1 @@
Subproject commit 9372f27a25d0718d3527afad6cc936f6a853f86e

1
vendor/nim-chronicles vendored Submodule

@ -0,0 +1 @@
Subproject commit 06445495141d3bef82ddd9e68dfd7887bdc46fbf

1
vendor/nim-chronos vendored Submodule

@ -0,0 +1 @@
Subproject commit 5b9ec0837c4f6774b68bd9e5ccc36bd79a65e57d

1
vendor/nim-datastore vendored Submodule

@ -0,0 +1 @@
Subproject commit 0cde8aeb67c59fd0ac95496dc6b5e1168d6632aa

1
vendor/nim-faststreams vendored Submodule

@ -0,0 +1 @@
Subproject commit 2a771bb91f8aae8520a5553955a2acce5fdd0c87

1
vendor/nim-http-utils vendored Submodule

@ -0,0 +1 @@
Subproject commit 5065d2cf18dcb9812e25cc0e2c50eb357bde04cf

1
vendor/nim-json-serialization vendored Submodule

@ -0,0 +1 @@
Subproject commit e32e8ad916ea12b8d356b2d691c4e2d8b8fea45d

1
vendor/nim-libp2p vendored Submodule

@ -0,0 +1 @@
Subproject commit 95e98e8c51d57d041a9094b300cd884b35e87b95

1
vendor/nim-libp2p-dht vendored Submodule

@ -0,0 +1 @@
Subproject commit 3afc0063423aa6b38a247a7e8399d381cdcd6f7c

1
vendor/nim-metrics vendored Submodule

@ -0,0 +1 @@
Subproject commit abf3acc7f06cee9ee2c287d2f31413dc3df4c04e

1
vendor/nim-secp256k1 vendored Submodule

@ -0,0 +1 @@
Subproject commit 05b4bde6d09b2c6ecc889bec2b1369a7264f3a29

1
vendor/nim-serialization vendored Submodule

@ -0,0 +1 @@
Subproject commit c23b49910db2dc20db9e731ec6421531b9379aaf

1
vendor/nim-sqlite3-abi vendored Submodule

@ -0,0 +1 @@
Subproject commit 362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3

1
vendor/nim-stew vendored Submodule

@ -0,0 +1 @@
Subproject commit ebbb391b9e87bb50907f8a17546cd383c41a713e

1
vendor/nim-websock vendored Submodule

@ -0,0 +1 @@
Subproject commit 2c3ae3137f3c9cb48134285bd4a47186fa51f0e8

1
vendor/nimbus-build-system vendored Submodule

@ -0,0 +1 @@
Subproject commit 239c3a7fbb88fd241da0ade3246fd2e5fcff4f25

1
vendor/nimcrypto vendored Submodule

@ -0,0 +1 @@
Subproject commit 4014ef939b51e02053c2e16dd3481d47bc9267dd

1
vendor/questionable vendored Submodule

@ -0,0 +1 @@
Subproject commit b18444a6d03293b0cd45c452fd0a7e1532c48c85

1
vendor/stint vendored Submodule

@ -0,0 +1 @@
Subproject commit c0ae9e10a9238883d18226fa28a5435c4d305e45

1
vendor/upraises vendored Submodule

@ -0,0 +1 @@
Subproject commit bc2628989b63854d980e92dadbd58f83e34b6f25