status-im/nim-libp2p
2
0
Fork 0
mirror of https://github.com/status-im/nim-libp2p.git synced 2025-02-22 17:48:12 +00:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/unstable' into quic

Browse Source
This commit is contained in:
Tanguy 2023-01-02 15:42:51 +01:00
commit af0a9ac66e
No known key found for this signature in database
GPG Key ID: 7DD8EC6B6CE6C45E
122 changed files with 5344 additions and 1411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
.github/actions/install_nim/action.yml vendored Normal file
View File

@ -0,0 +1,131 @@
name: Install Nim
inputs:
os:
description: "Operating system to build for"
required: true
cpu:
description: "CPU to build for"
default: "amd64"
nim_branch:
description: "Nim version"
default: "version-1-6"
shell:
description: "Shell to run commands in"
default: "bash --noprofile --norc -e -o pipefail"
runs:
using: "composite"
steps:
- name: Install build dependencies (Linux i386)
shell: ${{ inputs.shell }}
if: inputs.os == 'Linux' && inputs.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: inputs.os == 'Windows' && inputs.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: inputs.os == 'Windows' && inputs.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
install: >-
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: inputs.os == 'Windows'
id: windows-dlls-cache
uses: actions/cache@v3
with:
path: external/dlls
key: 'dlls'
- name: Install DLL dependencies (Windows)
shell: ${{ inputs.shell }}
if: >
steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
inputs.os == 'Windows'
run: |
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
shell: ${{ inputs.shell }}
if: >
inputs.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
shell: ${{ inputs.shell }}
run: |
if [[ '${{ inputs.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ inputs.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
- name: Restore Nim from cache
id: nim-cache
uses: actions/cache@v3
with:
path: '${{ github.workspace }}/nim'
key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_branch }}-cache-${{ env.cache_nonce }}
- name: Build Nim and Nimble
shell: ${{ inputs.shell }}
if: ${{ steps.nim-cache.outputs.cache-hit != 'true' }}
run: |
# We don't want partial matches of the cache restored
rm -rf nim
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ inputs.nim_branch }} \
QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
bash build_nim.sh nim csources dist/nimble NimBinaries

15
.github/workflows/bumper.yml vendored
View File

@ -7,14 +7,21 @@ on:
workflow_dispatch:
jobs:
bumpNimbus:
bumpProjects:
runs-on: ubuntu-latest
strategy:
matrix:
target: [
{ repo: status-im/nimbus-eth2, branch: unstable },
{ repo: status-im/nwaku, branch: master },
{ repo: status-im/nim-codex, branch: main }
]
steps:
- name: Clone NBC
- name: Clone repo
uses: actions/checkout@v2
with:
repository: status-im/nimbus-eth2
ref: unstable
repository: ${{ matrix.target.repo }}
ref: ${{ matrix.target.branch }}
path: nbc
submodules: true
fetch-depth: 0

126
.github/workflows/ci.yml vendored
View File

@ -7,6 +7,10 @@ on:
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build:
timeout-minutes: 90
@ -45,111 +49,20 @@ jobs:
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
runs-on: ${{ matrix.builder }}
continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }}
continue-on-error: ${{ matrix.branch == 'devel' }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
uses: msys2/setup-msys2@v2
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
install: >-
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: runner.os == 'Windows'
id: windows-dlls-cache
uses: actions/cache@v2
with:
path: external/dlls
key: 'dlls'
- name: Install DLL dependencies (Windows)
if: >
steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
runner.os == 'Windows'
run: |
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
if: >
runner.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
run: |
if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
- name: Build Nim and Nimble
run: |
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \
QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
bash build_nim.sh nim csources dist/nimble NimBinaries
echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
os: ${{ matrix.target.os }}
cpu: ${{ matrix.target.cpu }}
shell: ${{ matrix.shell }}
nim_branch: ${{ matrix.branch }}
- name: Setup Go
uses: actions/setup-go@v2
@ -160,13 +73,20 @@ jobs:
run: |
V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v3
with:
path: nimbledeps
key: nimbledeps-${{ hashFiles('.pinned') }}
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
nimble install_pinned
- name: Run tests
run: |
if [[ "${{ matrix.target.os }}" == "windows" ]]; then
# https://github.com/status-im/nimbus-eth2/issues/3121
export NIMFLAGS="-d:nimRawSetjmp"
fi
nim --version
nimble --version
nimble install_pinned
nimble test

151
.github/workflows/codecov.yml vendored
View File

@ -5,136 +5,61 @@ on:
push:
branches:
- master
- unstable
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
GossipSub:
Coverage:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
"",
"-d:libp2p_pubsub_anonymize=true -d:libp2p_pubsub_sign=false -d:libp2p_pubsub_verify=false",
"-d:libp2p_pubsub_sign=true -d:libp2p_pubsub_verify=true"
]
test-program: [
"tests/pubsub/testpubsub",
"tests/pubsub/testfloodsub",
"tests/pubsub/testgossipinternal"
]
env:
CICOV: YES
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
os: linux
cpu: amd64
shell: bash
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v3
with:
path: nimbledeps
key: nimbledeps-${{ hashFiles('.pinned') }}
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
nimble install_pinned
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install_pinned
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
export NIMFLAGS="--lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage"
nimble testnative
nimble testpubsub
nimble testfilter
find nimcache -name *.c -delete
lcov --capture --directory nimcache --output-file coverage/coverage.info
shopt -s globstar
ls `pwd`/libp2p/{*,**/*}.nim
lcov --extract coverage/coverage.info `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
genhtml coverage/coverage.f.info --output-directory coverage/output
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
Tests:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
""
]
test-program: [
"tests/testnative",
]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install_pinned
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }} --clearNimblePath --NimblePath:nimbledeps/pkgs"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
shopt -s globstar
ls `pwd`/libp2p/{*,**/*}.nim
lcov --extract coverage/coverage.info `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
Filter:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
"",
"-d:libp2p_pki_schemes=secp256k1",
"-d:libp2p_pki_schemes=secp256k1;ed25519",
"-d:libp2p_pki_schemes=secp256k1;ed25519;ecnist",
]
test-program: [
"tests/testpkifilter",
]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install_pinned
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
shopt -s globstar
ls `pwd`/libp2p/{*,**/*}.nim
lcov --extract coverage/coverage.info `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
#- uses: actions/upload-artifact@master
# with:
# name: coverage
# path: coverage

8
.github/workflows/doc.yml vendored
View File

@ -63,7 +63,7 @@ jobs:
git push origin gh-pages
update_site:
if: github.ref == 'refs/heads/master'
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs'
name: 'Rebuild website'
runs-on: ubuntu-latest
steps:
@ -74,8 +74,12 @@ jobs:
with:
python-version: 3.x
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: 'stable'
- name: Generate website
run: pip install mkdocs-material && mkdocs build
run: pip install mkdocs-material && nimble website
- name: Clone the gh-pages branch
uses: actions/checkout@v2

129
.github/workflows/multi_nim.yml vendored
View File

@ -5,7 +5,13 @@ on:
workflow_dispatch:
jobs:
delete-cache:
runs-on: ubuntu-latest
steps:
- uses: snnaplab/delete-branch-cache-action@v1
build:
needs: delete-cache
timeout-minutes: 120
strategy:
fail-fast: false
@ -21,7 +27,7 @@ jobs:
cpu: amd64
#- os: windows
#cpu: i386
branch: [version-1-2, version-1-4, version-1-6, devel]
branch: [version-1-2, version-1-6, devel]
include:
- target:
os: linux
@ -42,112 +48,18 @@ jobs:
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
runs-on: ${{ matrix.builder }}
continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }}
continue-on-error: ${{ matrix.branch == 'devel' }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
ref: unstable
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
install: >-
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: runner.os == 'Windows'
id: windows-dlls-cache
uses: actions/cache@v2
with:
path: external/dlls
key: 'dlls'
- name: Install DLL dependencies (Windows)
if: >
steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
runner.os == 'Windows'
run: |
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
if: >
runner.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
run: |
if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
- name: Build Nim and Nimble
run: |
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \
QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
bash build_nim.sh nim csources dist/nimble NimBinaries
echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
os: ${{ matrix.target.os }}
shell: ${{ matrix.shell }}
nim_branch: ${{ matrix.branch }}
cpu: ${{ matrix.target.cpu }}
- name: Setup Go
uses: actions/setup-go@v2
@ -160,16 +72,11 @@ jobs:
- name: Run tests
run: |
if [[ "${{ matrix.target.os }}" == "windows" ]]; then
# https://github.com/status-im/nimbus-eth2/issues/3121
export NIMFLAGS="-d:nimRawSetjmp"
fi
nim --version
nimble --version
nimble install -y --depsOnly
nimble test
if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then
NIMFLAGS="${NIMFLAGS} --gc:refc" nimble test
if [[ "${{ matrix.branch }}" == "devel" ]]; then
echo -e "\nTesting with '--gc:orc':\n"
export NIMFLAGS="${NIMFLAGS} --gc:orc"
nimble test
NIMFLAGS="${NIMFLAGS} --gc:orc" nimble test
fi

1
.gitignore vendored
View File

@ -13,5 +13,6 @@ build/
.vscode/
.DS_Store
tests/pubsub/testgossipsub
examples/*.md
nimble.develop
nimble.paths

26
.pinned
View File

@ -1,19 +1,19 @@
bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab
bearssl;https://github.com/status-im/nim-bearssl@#a647994910904b0103a05db3a5ec1ecfc4d91a88
chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
chronos;https://github.com/status-im/nim-chronos@#1334cdfebdc6182ff752e7d20796d9936cc8faa3
dnsclient;https://github.com/ba0f3/dnsclient.nim@#4960de2b345f567b12f09a08e9967af104ab39a3
faststreams;https://github.com/status-im/nim-faststreams@#49e2c52eb5dda46b1c9c10d079abe7bffe6cea89
httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040
json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4
metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301524871765f
chronos;https://github.com/status-im/nim-chronos@#e9f8baa6ee2e21ff8e6b6c0ce0c22368cdd9e758
dnsclient;https://github.com/ba0f3/dnsclient.nim@#fcd7443634b950eaea574e5eaa00a628ae029823
faststreams;https://github.com/status-im/nim-faststreams@#b42daf41d8eb4fbce40add6836bed838f8d85b6f
httputils;https://github.com/status-im/nim-http-utils@#a85bd52ae0a956983ca6b3267c72961d2ec0245f
json_serialization;https://github.com/status-im/nim-json-serialization@#a7d815ed92f200f490c95d3cfd722089cc923ce6
metrics;https://github.com/status-im/nim-metrics@#21e99a2e9d9f80e68bef65c80ef781613005fccb
ngtcp2;https://github.com/status-im/nim-ngtcp2@#76bf92475f55728ff55a2a19b45a5fcbb4faa2ab
nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178
quic;https://github.com/status-im/nim-quic.git@#26b1c76f8574883e385f8650090a74d33cc9d0fa
secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f
serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466
stew;https://github.com/status-im/nim-stew@#018760954a1530b7336aed7133393908875d860f
quic;https://github.com/status-im/nim-quic.git@#9dd9b528cfda50f6482f800175d7350cb03911ff
secp256k1;https://github.com/status-im/nim-secp256k1@#fd173fdff863ce2e211cf64c9a03bc7539fe40b0
serialization;https://github.com/status-im/nim-serialization@#d77417cba6896c26287a68e6a95762e45a1b87e5
stew;https://github.com/status-im/nim-stew@#f2f9685ec904868bbb48485e72ddc026ed51b230
testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34
unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db
unittest2;https://github.com/status-im/nim-unittest2@#da8398c45cafd5bd7772da1fc96e3924a18d3823
upraises;https://github.com/markspanbroek/upraises@#d9f268db1021959fe0f2c7a5e49fba741f9932a0
websock;https://github.com/status-im/nim-websock@#af8779d9d95e488ec9fd2d584b6328bd506c702b
websock;https://github.com/status-im/nim-websock@#691f069b209d372b1240d5ae1f57fb7bbafeaba7
zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8

87
README.md
View File

@ -2,7 +2,7 @@
<a href="https://libp2p.io"><img width="250" src="./.assets/full-logo.svg?raw=true" alt="nim-libp2p logo" /></a>
</h1>
<h3 align="center">The Nim implementation of the libp2p Networking Stack.</h3>
<h3 align="center">The <a href="https://nim-lang.org/">Nim</a> implementation of the <a href="https://libp2p.io/">libp2p</a> Networking Stack.</h3>
<p align="center">
<a href="https://github.com/status-im/nim-libp2p/actions"><img src="https://github.com/status-im/nim-libp2p/actions/workflows/ci.yml/badge.svg" /></a>
@ -16,30 +16,26 @@
<img src="https://img.shields.io/badge/nim-%3E%3D1.2.0-orange.svg?style=flat-square" />
</p>
## Introduction
An implementation of [libp2p](https://libp2p.io/) in [Nim](https://nim-lang.org/).
# Table of Contents
- [Background](#background)
- [Install](#install)
- [Getting Started](#getting-started)
- [Modules](#modules)
- [Users](#users)
- [Stability](#stability)
- [Development](#development)
- [Contribute](#contribute)
- [Core Developers](#core-developers)
- [Contributors](#contributors)
- [Core Maintainers](#core-maintainers)
- [License](#license)
## Background
libp2p is a networking stack and library modularized out of [The IPFS Project](https://github.com/ipfs/ipfs), and bundled separately for other tools to use.
libp2p is a [Peer-to-Peer](https://en.wikipedia.org/wiki/Peer-to-peer) networking stack, with [implementations](https://github.com/libp2p/libp2p#implementations) in multiple languages derived from the same [specifications.](https://github.com/libp2p/specs)
libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability.
Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It's striving to be a modular stack, with sane and secure defaults, useful protocols, while remain open and extensible.
This implementation in native Nim, relying on [chronos](https://github.com/status-im/nim-chronos) for async. It's used in production by a few [projects](#users)
libp2p grew out of IPFS, but it is built so that lots of people can use it, for lots of different projects.
- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io).
- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages.
Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow libp2p's documentation [**docs.libp2p.io**](https://docs.libp2p.io).
## Install
**Prerequisite**
@ -49,7 +45,7 @@ nimble install libp2p
```
## Getting Started
You'll find the documentation [here].(https://status-im.github.io/nim-libp2p/docs/)
You'll find the nim-libp2p documentation [here](https://status-im.github.io/nim-libp2p/docs/).
**Go Daemon:**
Please find the installation and usage intructions in [daemonapi.md](examples/go-daemon/daemonapi.md).
@ -63,25 +59,28 @@ List of packages modules implemented in nim-libp2p:
| **Libp2p** | |
| [libp2p](libp2p/switch.nim) | The core of the project |
| [connmanager](libp2p/connmanager.nim) | Connection manager |
| [identify / push identify](libp2p/protocols/identify.nim) | [Identify](https://docs.libp2p.io/concepts/protocols/#identify) protocol |
| [ping](libp2p/protocols/ping.nim) | [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol |
| [identify / push identify](libp2p/protocols/identify.nim) | [Identify](https://docs.libp2p.io/concepts/fundamentals/protocols/#identify) protocol |
| [ping](libp2p/protocols/ping.nim) | [Ping](https://docs.libp2p.io/concepts/fundamentals/protocols/#ping) protocol |
| [libp2p-daemon-client](libp2p/daemon/daemonapi.nim) | [go-daemon](https://github.com/libp2p/go-libp2p-daemon) nim wrapper |
| [interop-libp2p](tests/testinterop.nim) | Interop tests |
| **Transports** | |
| [libp2p-tcp](libp2p/transports/tcptransport.nim) | TCP transport |
| [libp2p-ws](libp2p/transports/wstransport.nim) | WebSocket & WebSocket Secure transport |
| [libp2p-tor](libp2p/transports/tortransport.nim) | Tor Transport |
| **Secure Channels** | |
| [libp2p-secio](libp2p/protocols/secure/secio.nim) | [Secio](https://docs.libp2p.io/concepts/protocols/#secio) secure channel |
| [libp2p-noise](libp2p/protocols/secure/noise.nim) | [Noise](https://github.com/libp2p/specs/tree/master/noise) secure channel |
| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim) | [Plain Text](https://github.com/libp2p/specs/tree/master/plaintext) for development purposes |
| [libp2p-secio](libp2p/protocols/secure/secio.nim) | Secio secure channel |
| [libp2p-noise](libp2p/protocols/secure/noise.nim) | [Noise](https://docs.libp2p.io/concepts/secure-comm/noise/) secure channel |
| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim) | Plain Text for development purposes |
| **Stream Multiplexers** | |
| [libp2p-mplex](libp2p/muxers/mplex/mplex.nim) | [MPlex](https://github.com/libp2p/specs/tree/master/mplex) multiplexer |
| [libp2p-yamux](libp2p/muxers/yamux/yamux.nim) | [Yamux](https://docs.libp2p.io/concepts/multiplex/yamux/) multiplexer |
| **Data Types** | |
| [peer-id](libp2p/peerid.nim) | [Cryptographic identifiers](https://docs.libp2p.io/concepts/peer-id/) |
| [peer-store](libp2p/peerstore.nim) | ["Phone book" of known peers](https://docs.libp2p.io/concepts/peer-id/#peerinfo) |
| [peer-id](libp2p/peerid.nim) | [Cryptographic identifiers](https://docs.libp2p.io/concepts/fundamentals/peers/#peer-id) |
| [peer-store](libp2p/peerstore.nim) | ["Address book" of known peers](https://docs.libp2p.io/concepts/fundamentals/peers/#peer-store) |
| [multiaddress](libp2p/multiaddress.nim) | [Composable network addresses](https://github.com/multiformats/multiaddr) |
| [signed envelope](libp2p/signed_envelope.nim) | [Signed generic data container](https://github.com/libp2p/specs/blob/master/RFC/0002-signed-envelopes.md) |
| [routing record](libp2p/routing_record.nim) | [Signed peer dialing informations](https://github.com/libp2p/specs/blob/master/RFC/0003-routing-records.md) |
| [discovery manager](libp2p/discovery/discoverymngr.nim) | Discovery Manager |
| **Utilities** | |
| [libp2p-crypto](libp2p/crypto) | Cryptographic backend |
| [libp2p-crypto-secp256k1](libp2p/crypto/secp.nim) | |
@ -98,20 +97,34 @@ nim-libp2p is used by:
- [nim-codex](https://github.com/status-im/nim-codex), a decentralized storage application
- (open a pull request if you want to be included here)
## Development
**Clone and Install dependencies:**
## Stability
nim-libp2p has been used in production for over a year in high-stake scenarios, so its core is considered stable.
Some modules are more recent and less stable.
The versioning follows [semver](https://semver.org/), with some additions:
- Some of libp2p procedures are marked as `.public.`, they will remain compatible during each `MAJOR` version
- The rest of the procedures are considered internal, and can change at any `MINOR` version (but remain compatible for each new `PATCH`)
We aim to be compatible at all time with at least 2 Nim `MINOR` versions, currently `1.2 & 1.6`
## Development
Clone and Install dependencies:
```sh
git clone https://github.com/status-im/nim-libp2p
cd nim-libp2p
nimble install
# to use dependencies computed by nimble
nimble install -dy
# OR to install the dependencies versions used in CI
nimble install_pinned
```
**Run unit tests**
Run unit tests:
```sh
# run all the unit tests
nimble test
```
This requires the go daemon to be available. To only run native tests, use `nimble testnative`.
Or use `nimble tasks` to show all available tasks.
### Contribute
@ -122,25 +135,33 @@ The libp2p implementation in Nim is a work in progress. We welcome contributors
The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
### Core Developers
[@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Tanguy](https://github.com/Menduist), [Zahary Karadjov](https://github.com/zah)
### Contributors
<a href="https://github.com/status-im/nim-libp2p/graphs/contributors"><img src="https://contrib.rocks/image?repo=status-im/nim-libp2p" alt="nim-libp2p contributors"></a>
### Tips and tricks
### Core Maintainers
<table>
<tbody>
<tr>
<td align="center"><a href="https://github.com/Menduist"><img src="https://avatars.githubusercontent.com/u/13471753?v=4?s=100" width="100px;" alt="Tanguy"/><br /><sub><b>Tanguy (Menduist)</b></sub></a></td>
<td align="center"><a href="https://github.com/lchenut"><img src="https://avatars.githubusercontent.com/u/11214565?v=4?s=100" width="100px;" alt="Ludovic"/><br /><sub><b>Ludovic</b></sub></a></td>
<td align="center"><a href="https://github.com/diegomrsantos"><img src="https://avatars.githubusercontent.com/u/7316595?v=4?s=100" width="100px;" alt="Diego"/><br /><sub><b>Diego</b></sub></a></td>
</tr>
</tbody>
</table>
**enable expensive metrics:**
### Compile time flags
Enable expensive metrics (ie, metrics with per-peer cardinality):
```bash
nim c -d:libp2p_expensive_metrics some_file.nim
```
**use identify metrics**
Set list of known libp2p agents for metrics:
```bash
nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku some_file.nim
```
**specify gossipsub specific topics to measure**
Specify gossipsub specific topics to measure in the metrics:
```bash
nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
```

8
codecov.yml
View File

@ -1,14 +1,8 @@
codecov:
notify:
require_ci_to_pass: true
# must be the number of coverage report builds
# notice that this number is for PRs;
# like this we disabled notify on pure branches report
# which is fine I guess
after_n_builds: 28
comment:
layout: "reach, diff, flags, files"
after_n_builds: 28 # must be the number of coverage report builds
coverage:
status:
project:
@ -16,4 +10,4 @@ coverage:
# basic settings
target: auto
threshold: 5%
base: auto
base: auto

15
config.nims
View File

@ -2,6 +2,21 @@
if dirExists("nimbledeps/pkgs"):
switch("NimblePath", "nimbledeps/pkgs")
switch("warning", "CaseTransition:off")
switch("warning", "ObservableStores:off")
switch("warning", "LockLevel:off")
--define:chronosStrictException
--styleCheck:usages
if (NimMajor, NimMinor) < (1, 6):
--styleCheck:hint
else:
--styleCheck:error
# Avoid some rare stack corruption while using exceptions with a SEH-enabled
# toolchain: https://github.com/status-im/nimbus-eth2/issues/3121
if defined(windows) and not defined(vcc):
--define:nimRawSetjmp
# begin Nimble config (version 1)
when fileExists("nimble.paths"):
include "nimble.paths"

7
examples/README.md
View File

@ -1,6 +1,5 @@
# nim-libp2p documentation
# nim-libp2p examples
Welcome to the nim-libp2p documentation!
In this folder, you'll find the sources of the [nim-libp2p website](https://status-im.github.io/nim-libp2p/docs/)
Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as [examples](directchat.nim) and
the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).
We recommand to follow the tutorials on the website, but feel free to grok the sources here!

26
examples/circuitrelay.nim
View File

@ -1,6 +1,14 @@
## # Circuit Relay example
##
## Circuit Relay can be used when a node cannot reach another node
## directly, but can reach it through a another node (the Relay).
##
## That may happen because of NAT, Firewalls, or incompatible transports.
##
## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/).
import chronos, stew/byteutils
import ../libp2p,
../libp2p/protocols/relay/[relay, client]
import libp2p,
libp2p/protocols/connectivity/relay/[relay, client]
# Helper to create a circuit relay node
proc createCircuitRelaySwitch(r: Relay): Switch =
@ -40,19 +48,19 @@ proc main() {.async.} =
swSrc = createCircuitRelaySwitch(clSrc)
swDst = createCircuitRelaySwitch(clDst)
# Create a relay address to swDst using swRel as the relay
addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
$swRel.peerInfo.peerId & "/p2p-circuit/p2p/" &
$swDst.peerInfo.peerId).get()
swDst.mount(proto)
await swRel.start()
await swSrc.start()
await swDst.start()
# Connect both Src and Dst to the relay, but not to each other.
await swSrc.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
let
# Create a relay address to swDst using swRel as the relay
addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
$swRel.peerInfo.peerId & "/p2p-circuit/p2p/" &
$swDst.peerInfo.peerId).get()
# Connect Dst to the relay
await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
# Dst reserve a slot on the relay.

2
examples/directchat.nim
View File

@ -5,7 +5,7 @@ import
strformat, strutils,
stew/byteutils,
chronos,
../libp2p
libp2p
const DefaultAddr = "/ip4/127.0.0.1/tcp/0"

2
examples/helloworld.nim
View File

@ -1,6 +1,6 @@
import chronos # an efficient library for async
import stew/byteutils # various utils
import ../libp2p # when installed through nimble, just use `import libp2p`
import libp2p
##
# Create our custom protocol

6
examples/index.md Normal file
View File

@ -0,0 +1,6 @@
# nim-libp2p documentation
Welcome to the nim-libp2p documentation!
Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as
the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).

108
examples/tutorial_1_connect.md
View File

@ -1,108 +0,0 @@
# Simple ping tutorial
Hi all, welcome to the first nim-libp2p tutorial!
!!! tips ""
This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
Hope you'll find it helpful in your journey of learning. Happy coding! ;)
## Before you start
The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
Install Nim via their [official website](https://nim-lang.org/install.html).
Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
You can now install the latest version of `nim-libp2p`:
```bash
nimble install libp2p@#master
```
## A simple ping application
We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
!!! tips ""
You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder!
Let's create a `part1.nim`, and import our dependencies:
```nim
import chronos
import libp2p
import libp2p/protocols/ping
```
[chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
```nim
proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
var switch = SwitchBuilder
.new()
.withRng(rng) # Give the application RNG
.withAddress(ma) # Our local address(es)
.withTcpTransport() # Use TCP as transport
.withMplex() # Use Mplex as muxer
.withNoise() # Use Noise as secure manager
.build()
return switch
```
This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
Let's now start to create our main procedure:
```nim
proc main() {.async, gcsafe.} =
let
rng = newRng()
localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
pingProtocol = Ping.new(rng=rng)
```
We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
`tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
We can now create our two switches:
```nim
let
switch1 = createSwitch(localAddress, rng)
switch2 = createSwitch(localAddress, rng)
switch1.mount(pingProtocol)
await switch1.start()
await switch2.start()
```
We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
Now that we've started the nodes, they are listening for incoming peers.
We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
```nim
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
```
We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
```nim
# ping the other node and echo the ping duration
echo "ping: ", await pingProtocol.ping(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
```
And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
```nim
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
waitFor(main())
```
You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

95
examples/tutorial_1_connect.nim Normal file
View File

@ -0,0 +1,95 @@
## # Simple ping tutorial
##
## Hi all, welcome to the first nim-libp2p tutorial!
##
## !!! tips ""
## This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
##
## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
##
## Hope you'll find it helpful in your journey of learning. Happy coding! ;)
##
## ## Before you start
## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
##
## Install Nim via their [official website](https://nim-lang.org/install.html).
## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
##
## You can now install the latest version of `nim-libp2p`:
## ```bash
## nimble install libp2p@#master
## ```
##
## ## A simple ping application
## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
##
## !!! tips ""
## You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder!
##
## Let's create a `part1.nim`, and import our dependencies:
import chronos
import libp2p
import libp2p/protocols/ping
## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
##
## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
var switch = SwitchBuilder
.new()
.withRng(rng) # Give the application RNG
.withAddress(ma) # Our local address(es)
.withTcpTransport() # Use TCP as transport
.withMplex() # Use Mplex as muxer
.withNoise() # Use Noise as secure manager
.build()
return switch
## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
##
## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
##
##
## Let's now start to create our main procedure:
proc main() {.async, gcsafe.} =
let
rng = newRng()
localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
pingProtocol = Ping.new(rng=rng)
## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
##
## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
##
## We can now create our two switches:
let
switch1 = createSwitch(localAddress, rng)
switch2 = createSwitch(localAddress, rng)
switch1.mount(pingProtocol)
await switch1.start()
await switch2.start()
## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
##
## Now that we've started the nodes, they are listening for incoming peers.
## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
##
## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
# ping the other node and echo the ping duration
echo "ping: ", await pingProtocol.ping(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
waitFor(main())
## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
##
## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

82
examples/tutorial_2_customproto.md
View File

@ -1,82 +0,0 @@
# Custom protocol in libp2p
In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
We'll now look at how to create a custom protocol inside the libp2p
Let's create a `part2.nim`, and import our dependencies:
```nim
import chronos
import stew/byteutils
import libp2p
```
This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
Next, we'll declare our custom protocol
```nim
const TestCodec = "/test/proto/1.0.0"
type TestProto = ref object of LPProtocol
```
We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
A protocol generally has two part: and handling/server part, and a dialing/client part.
Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
Let's start with the server part:
```nim
proc new(T: typedesc[TestProto]): T =
# every incoming connections will in be handled in this closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
# Read up to 1024 bytes from this connection, and transform them into
# a string
echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
# We must close the connections ourselves when we're done with it
await conn.close()
return T(codecs: @[TestCodec], handler: handle)
```
This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
In our handle, we simply read a message from the connection and `echo` it.
We can now create our client part:
```nim
proc hello(p: TestProto, conn: Connection) {.async.} =
await conn.writeLp("Hello p2p!")
```
Again, pretty straight-forward, we just send a message on the connection.
We can now create our main procedure:
```nim
proc main() {.async, gcsafe.} =
let
rng = newRng()
testProto = TestProto.new()
switch1 = newStandardSwitch(rng=rng)
switch2 = newStandardSwitch(rng=rng)
switch1.mount(testProto)
await switch1.start()
await switch2.start()
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
await testProto.hello(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
```
This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
We can now wrap our program by calling our main proc:
```nim
waitFor(main())
```
And that's it!

74
examples/tutorial_2_customproto.nim Normal file
View File

@ -0,0 +1,74 @@
## # Custom protocol in libp2p
##
## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
##
## We'll now look at how to create a custom protocol inside the libp2p
##
## Let's create a `part2.nim`, and import our dependencies:
import chronos
import stew/byteutils
import libp2p
## This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
##
## Next, we'll declare our custom protocol
const TestCodec = "/test/proto/1.0.0"
type TestProto = ref object of LPProtocol
## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
##
## A protocol generally has two part: and handling/server part, and a dialing/client part.
## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
##
## Let's start with the server part:
proc new(T: typedesc[TestProto]): T =
# every incoming connections will in be handled in this closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
# Read up to 1024 bytes from this connection, and transform them into
# a string
echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
# We must close the connections ourselves when we're done with it
await conn.close()
return T.new(codecs = @[TestCodec], handler = handle)
## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
## In our handle, we simply read a message from the connection and `echo` it.
##
## We can now create our client part:
proc hello(p: TestProto, conn: Connection) {.async.} =
await conn.writeLp("Hello p2p!")
## Again, pretty straight-forward, we just send a message on the connection.
##
## We can now create our main procedure:
proc main() {.async, gcsafe.} =
let
rng = newRng()
testProto = TestProto.new()
switch1 = newStandardSwitch(rng=rng)
switch2 = newStandardSwitch(rng=rng)
switch1.mount(testProto)
await switch1.start()
await switch2.start()
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
await testProto.hello(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
##
## We can now wrap our program by calling our main proc:
waitFor(main())
## And that's it!
## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf.

162
examples/tutorial_3_protobuf.nim Normal file
View File

@ -0,0 +1,162 @@
## # Protobuf usage
##
## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol.
## Most real protocol want their messages to be structured and extensible, which is why
## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to
## define their message structures.
##
## Here, we'll create a slightly more complex protocol, which parses & generate protobuf
## messages. Let's start by importing our dependencies, as usual:
import chronos
import stew/results # for Opt[T]
import libp2p
## ## Protobuf encoding & decoding
## This will be the structure of our messages:
## ```protobuf
## message MetricList {
## message Metric {
## string name = 1;
## float value = 2;
## }
##
## repeated Metric metrics = 2;
## }
## ```
## We'll create our protobuf types, encoders & decoders, according to this format.
## To create the encoders & decoders, we are going to use minprotobuf
## (included in libp2p).
##
## While more modern technics
## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization))
## exists, minprotobuf is currently the recommended method to handle protobuf, since it has
## been used in production extensively, and audited.
type
Metric = object
name: string
value: float
MetricList = object
metrics: seq[Metric]
{.push raises: [].}
proc encode(m: Metric): ProtoBuffer =
result = initProtoBuffer()
result.write(1, m.name)
result.write(2, m.value)
result.finish()
proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
var res: Metric
let pb = initProtoBuffer(buf)
# "getField" will return a Result[bool, ProtoError].
# The Result will hold an error if the protobuf is invalid.
# The Result will hold "false" if the field is missing
#
# We are just checking the error, and ignoring whether the value
# is present or not (default values are valid).
discard ? pb.getField(1, res.name)
discard ? pb.getField(2, res.value)
ok(res)
proc encode(m: MetricList): ProtoBuffer =
result = initProtoBuffer()
for metric in m.metrics:
result.write(1, metric.encode())
result.finish()
proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
var
res: MetricList
metrics: seq[seq[byte]]
let pb = initProtoBuffer(buf)
discard ? pb.getRepeatedField(1, metrics)
for metric in metrics:
res.metrics &= ? Metric.decode(metric)
ok(res)
## ## Results instead of exceptions
## As you can see, this part of the program also uses Results instead of exceptions for error handling.
## We start by `{.push raises: [].}`, which will prevent every non-async function from raising
## exceptions.
##
## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey
## errors to function callers. A `Result[T, E]` will either hold a valid result of type
## T, or an error of type E.
##
## You can check if the call succeeded by using `res.isOk`, and then get the
## value using `res.value` or the error by using `res.error`.
##
## Another useful tool is `?`, which will unpack a Result if it succeeded,
## or if it failed, exit the current procedure returning the error.
##
## nim-result is packed with other functionalities that you'll find in the
## nim-result repository.
##
## Results and exception are generally interchangeable, but have different semantics
## that you may or may not prefer.
##
## ## Creating the protocol
## We'll next create a protocol, like in the last tutorial, to request these metrics from our host
type
MetricCallback = proc: Future[MetricList] {.raises: [], gcsafe.}
MetricProto = ref object of LPProtocol
metricGetter: MetricCallback
proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto =
var res: MetricProto
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
let
metrics = await res.metricGetter()
asProtobuf = metrics.encode()
await conn.writeLp(asProtobuf.buffer)
await conn.close()
res = MetricProto.new(@["/metric-getter/1.0.0"], handle)
res.metricGetter = cb
return res
proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} =
let protobuf = await conn.readLp(2048)
# tryGet will raise an exception if the Result contains an error.
# It's useful to bridge between exception-world and result-world
return MetricList.decode(protobuf).tryGet()
## We can now create our main procedure:
proc main() {.async, gcsafe.} =
let rng = newRng()
proc randomMetricGenerator: Future[MetricList] {.async.} =
let metricCount = rng[].generate(uint32) mod 16
for i in 0 ..< metricCount + 1:
result.metrics.add(Metric(
name: "metric_" & $i,
value: float(rng[].generate(uint16)) / 1000.0
))
return result
let
metricProto1 = MetricProto.new(randomMetricGenerator)
metricProto2 = MetricProto.new(randomMetricGenerator)
switch1 = newStandardSwitch(rng=rng)
switch2 = newStandardSwitch(rng=rng)
switch1.mount(metricProto1)
await switch1.start()
await switch2.start()
let
conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs)
metrics = await metricProto2.fetch(conn)
await conn.close()
for metric in metrics.metrics:
echo metric.name, " = ", metric.value
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
waitFor(main())
## If you run this program, you should see random metrics being sent from the switch1 to the switch2.

163
examples/tutorial_4_gossipsub.nim Normal file
View File

@ -0,0 +1,163 @@
## # GossipSub
##
## In this tutorial, we'll build a simple GossipSub network
## to broadcast the metrics we built in the previous tutorial.
##
## GossipSub is used to broadcast some messages in a network,
## and allows to balance between latency, bandwidth usage,
## privacy and attack resistance.
##
## You'll find a good explanation on how GossipSub works
## [here.](https://docs.libp2p.io/concepts/publish-subscribe/) There are a lot
## of parameters you can tweak to adjust how GossipSub behaves but here we'll
## use the sane defaults shipped with libp2p.
##
## We'll start by creating our metric structure like previously
import chronos
import stew/results
import libp2p
import libp2p/protocols/pubsub/rpc/messages
type
Metric = object
name: string
value: float
MetricList = object
hostname: string
metrics: seq[Metric]
{.push raises: [].}
proc encode(m: Metric): ProtoBuffer =
result = initProtoBuffer()
result.write(1, m.name)
result.write(2, m.value)
result.finish()
proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
var res: Metric
let pb = initProtoBuffer(buf)
discard ? pb.getField(1, res.name)
discard ? pb.getField(2, res.value)
ok(res)
proc encode(m: MetricList): ProtoBuffer =
result = initProtoBuffer()
for metric in m.metrics:
result.write(1, metric.encode())
result.write(2, m.hostname)
result.finish()
proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
var
res: MetricList
metrics: seq[seq[byte]]
let pb = initProtoBuffer(buf)
discard ? pb.getRepeatedField(1, metrics)
for metric in metrics:
res.metrics &= ? Metric.decode(metric)
? pb.getRequiredField(2, res.hostname)
ok(res)
## This is exactly like the previous structure, except that we added
## a `hostname` to distinguish where the metric is coming from.
##
## Now we'll create a small GossipSub network to broadcast the metrics,
## and collect them on one of the node.
type Node = tuple[switch: Switch, gossip: GossipSub, hostname: string]
proc oneNode(node: Node, rng: ref HmacDrbgContext) {.async.} =
# This procedure will handle one of the node of the network
node.gossip.addValidator(["metrics"],
proc(topic: string, message: Message): Future[ValidationResult] {.async.} =
let decoded = MetricList.decode(message.data)
if decoded.isErr: return ValidationResult.Reject
return ValidationResult.Accept
)
# This "validator" will attach to the `metrics` topic and make sure
# that every message in this topic is valid. This allows us to stop
# propagation of invalid messages quickly in the network, and punish
# peers sending them.
# `John` will be responsible to log the metrics, the rest of the nodes
# will just forward them in the network
if node.hostname == "John":
node.gossip.subscribe("metrics",
proc (topic: string, data: seq[byte]) {.async.} =
echo MetricList.decode(data).tryGet()
)
else:
node.gossip.subscribe("metrics", nil)
# Create random metrics 10 times and broadcast them
for _ in 0..<10:
await sleepAsync(500.milliseconds)
var metricList = MetricList(hostname: node.hostname)
let metricCount = rng[].generate(uint32) mod 4
for i in 0 ..< metricCount + 1:
metricList.metrics.add(Metric(
name: "metric_" & $i,
value: float(rng[].generate(uint16)) / 1000.0
))
discard await node.gossip.publish("metrics", encode(metricList).buffer)
await node.switch.stop()
## For our main procedure, we'll create a few nodes, and connect them together.
## Note that they are not all interconnected, but GossipSub will take care of
## broadcasting to the full network nonetheless.
proc main {.async.} =
let rng = newRng()
var nodes: seq[Node]
for hostname in ["John", "Walter", "David", "Thuy", "Amy"]:
let
switch = newStandardSwitch(rng=rng)
gossip = GossipSub.init(switch = switch, triggerSelf = true)
switch.mount(gossip)
await switch.start()
nodes.add((switch, gossip, hostname))
for index, node in nodes:
# Connect to a few neighbors
for otherNodeIdx in index - 1 .. index + 2:
if otherNodeIdx notin 0 ..< nodes.len or otherNodeIdx == index: continue
let otherNode = nodes[otherNodeIdx]
await node.switch.connect(
otherNode.switch.peerInfo.peerId,
otherNode.switch.peerInfo.addrs)
var allFuts: seq[Future[void]]
for node in nodes:
allFuts.add(oneNode(node, rng))
await allFutures(allFuts)
waitFor(main())
## If you run this program, you should see something like:
## ```
## (hostname: "John", metrics: @[(name: "metric_0", value: 42.097), (name: "metric_1", value: 50.99), (name: "metric_2", value: 47.86), (name: "metric_3", value: 5.368)])
## (hostname: "Walter", metrics: @[(name: "metric_0", value: 39.452), (name: "metric_1", value: 15.606), (name: "metric_2", value: 14.059), (name: "metric_3", value: 6.68)])
## (hostname: "David", metrics: @[(name: "metric_0", value: 9.82), (name: "metric_1", value: 2.862), (name: "metric_2", value: 15.514)])
## (hostname: "Thuy", metrics: @[(name: "metric_0", value: 59.038)])
## (hostname: "Amy", metrics: @[(name: "metric_0", value: 55.616), (name: "metric_1", value: 23.52), (name: "metric_2", value: 59.081), (name: "metric_3", value: 2.516)])
## ```
##
## This is John receiving & logging everyone's metrics.
##
## ## Going further
## Building efficient & safe GossipSub networks is a tricky subject. By tweaking the [gossip params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#GossipSubParams)
## and [topic params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#TopicParams),
## you can achieve very different properties.
##
## Also see reports for [GossipSub v1.1](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4)
##
## If you are interested in broadcasting for your application, you may want to use [Waku](https://waku.org/), which builds on top of GossipSub,
## and adds features such as history, spam protection, and light node friendliness.

132
examples/tutorial_5_discovery.nim Normal file
View File

@ -0,0 +1,132 @@
## # Discovery Manager
##
## In the [previous tutorial](tutorial_4_gossipsub.md), we built a custom protocol using [protobuf](https://developers.google.com/protocol-buffers) and
## spread informations (some metrics) on the network using gossipsub.
## For this tutorial, on the other hand, we'll go back on a simple example
## we'll try to discover a specific peers to greet on the network.
##
## First, as usual, we import the dependencies:
import sequtils
import chronos
import stew/byteutils
import libp2p
import libp2p/protocols/rendezvous
import libp2p/discovery/rendezvousinterface
import libp2p/discovery/discoverymngr
## We'll not use newStandardSwitch this time as we need the discovery protocol
## [RendezVous](https://github.com/libp2p/specs/blob/master/rendezvous/README.md) to be mounted on the switch using withRendezVous.
##
## Note that other discovery methods such as [Kademlia](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) or [discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) exist.
proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withYamux()
.withNoise()
.withRendezVous(rdv)
.build()
# Create a really simple protocol to log one message received then close the stream
const DumbCodec = "/dumb/proto/1.0.0"
type DumbProto = ref object of LPProtocol
proc new(T: typedesc[DumbProto], nodeNumber: int): T =
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
echo "Node", nodeNumber, " received: ", string.fromBytes(await conn.readLp(1024))
await conn.close()
return T.new(codecs = @[DumbCodec], handler = handle)
## ## Bootnodes
## The first time a p2p program is ran, he needs to know how to join
## its network. This is generally done by hard-coding a list of stable
## nodes in the binary, called "bootnodes". These bootnodes are a
## critical part of a p2p network, since they are used by every new
## user to onboard the network.
##
## By using libp2p, we can use any node supporting our discovery protocol
## (rendezvous in this case) as a bootnode. For this example, we'll
## create a bootnode, and then every peer will advertise itself on the
## bootnode, and use it to find other peers
proc main() {.async, gcsafe.} =
let bootNode = createSwitch()
await bootNode.start()
# Create 5 nodes in the network
var
switches: seq[Switch] = @[]
discManagers: seq[DiscoveryManager] = @[]
for i in 0..5:
let rdv = RendezVous.new()
# Create a remote future to await at the end of the program
let switch = createSwitch(rdv)
switch.mount(DumbProto.new(i))
switches.add(switch)
# A discovery manager is a simple tool, you can set it up by adding discovery
# interfaces (such as RendezVousInterface) then you can use it to advertise
# something on the network or to request something from it.
let dm = DiscoveryManager()
# A RendezVousInterface is a RendezVous protocol wrapped to be usable by the
# DiscoveryManager.
dm.add(RendezVousInterface.new(rdv))
discManagers.add(dm)
# We can now start the switch and connect to the bootnode
await switch.start()
await switch.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
# Each nodes of the network will advertise on some topics (EvenGang or OddClub)
dm.advertise(RdvNamespace(if i mod 2 == 0: "EvenGang" else: "OddClub"))
## We can now create the newcomer. This peer will connect to the boot node, and use
## it to discover peers & greet them.
let
rdv = RendezVous.new()
newcomer = createSwitch(rdv)
dm = DiscoveryManager()
await newcomer.start()
await newcomer.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
dm.add(RendezVousInterface.new(rdv, ttr = 250.milliseconds))
# Use the discovery manager to find peers on the OddClub topic to greet them
let queryOddClub = dm.request(RdvNamespace("OddClub"))
for _ in 0..2:
let
# getPeer give you a PeerAttribute containing informations about the peer.
res = await queryOddClub.getPeer()
# Here we will use the PeerId and the MultiAddress to greet him
conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
await conn.writeLp("Odd Club suuuucks! Even Gang is better!")
# Uh-oh!
await conn.close()
# Wait for the peer to close the stream
await conn.join()
# Queries will run in a loop, so we must stop them when we are done
queryOddClub.stop()
# Maybe it was because he wanted to join the EvenGang
let queryEvenGang = dm.request(RdvNamespace("EvenGang"))
for _ in 0..2:
let
res = await queryEvenGang.getPeer()
conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
await conn.writeLp("Even Gang is sooo laaame! Odd Club rocks!")
# Or maybe not...
await conn.close()
await conn.join()
queryEvenGang.stop()
# What can I say, some people just want to watch the world burn... Anyway
# Stop all the discovery managers
for d in discManagers:
d.stop()
dm.stop()
# Stop all the switches
await allFutures(switches.mapIt(it.stop()))
await allFutures(bootNode.stop(), newcomer.stop())
waitFor(main())

259
examples/tutorial_6_game.nim Normal file
View File

@ -0,0 +1,259 @@
## # Tron example
##
## In this tutorial, we will create a video game based on libp2p, using
## all of the features we talked about in the last tutorials.
##
## We will:
## - Discover peers using the Discovery Manager
## - Use GossipSub to find a play mate
## - Create a custom protocol to play with him
##
## While this may look like a daunting project, it's less than 150 lines of code.
##
## The game will be a simple Tron. We will use [nico](https://github.com/ftsf/nico)
## as a game engine. (you need to run `nimble install nico` to have it available)
##
## ![multiplay](https://user-images.githubusercontent.com/13471753/198852714-b55048e3-f233-4723-900d-2193ad259fe1.gif)
##
## We will start by importing our dependencies and creating our types
import os
import nico, chronos, stew/byteutils, stew/endians2
import libp2p
import libp2p/protocols/rendezvous
import libp2p/discovery/rendezvousinterface
import libp2p/discovery/discoverymngr
const
directions = @[(K_UP, 0, -1), (K_LEFT, -1, 0), (K_DOWN, 0, 1), (K_RIGHT, 1, 0)]
mapSize = 32
tickPeriod = 0.2
type
Player = ref object
x, y: int
currentDir, nextDir: int
lost: bool
color: int
Game = ref object
gameMap: array[mapSize * mapSize, int]
tickTime: float
localPlayer, remotePlayer: Player
peerFound: Future[Connection]
hasCandidate: bool
tickFinished: Future[int]
GameProto = ref object of LPProtocol
proc new(_: type[Game]): Game =
# Default state of a game
result = Game(
tickTime: -3.0, # 3 seconds of "warm-up" time
localPlayer: Player(x: 4, y: 16, currentDir: 3, nextDir: 3, color: 8),
remotePlayer: Player(x: 27, y: 16, currentDir: 1, nextDir: 1, color: 12),
peerFound: newFuture[Connection]()
)
for pos in 0 .. result.gameMap.high:
if pos mod mapSize in [0, mapSize - 1] or pos div mapSize in [0, mapSize - 1]:
result.gameMap[pos] = 7
## ## Game Logic
## The networking during the game will work like this:
##
## * Each player will have `tickPeriod` (0.1) seconds to choose
## a direction that he wants to go to (default to current direction)
## * After `tickPeriod`, we will send our choosen direction to the peer,
## and wait for his direction
## * Once we have both direction, we will "tick" the game, and restart the
## loop, as long as both player are alive.
##
## This is a very simplistic scheme, but creating proper networking for
## video games is an [art](https://developer.valvesoftware.com/wiki/Latency_Compensating_Methods_in_Client/Server_In-game_Protocol_Design_and_Optimization)
##
## The main drawback of this scheme is that the more ping you have with
## the peer, the slower the game will run. Or invertedly, the less ping you
## have, the faster it runs!
proc update(g: Game, dt: float32) =
# Will be called at each frame of the game.
#
# Because both Nico and Chronos have a main loop,
# they must share the control of the main thread.
# This is a hacky way to make this happen
waitFor(sleepAsync(1.milliseconds))
# Don't do anything if we are still waiting for an opponent
if not(g.peerFound.finished()) or isNil(g.tickFinished): return
g.tickTime += dt
# Update the wanted direction, making sure we can't go backward
for i in 0 .. directions.high:
if i != (g.localPlayer.currentDir + 2 mod 4) and keyp(directions[i][0]):
g.localPlayer.nextDir = i
if g.tickTime > tickPeriod and not g.tickFinished.finished():
# We choosen our next direction, let the networking know
g.localPlayer.currentDir = g.localPlayer.nextDir
g.tickFinished.complete(g.localPlayer.currentDir)
proc tick(g: Game, p: Player) =
# Move player and check if he lost
p.x += directions[p.currentDir][1]
p.y += directions[p.currentDir][2]
if g.gameMap[p.y * mapSize + p.x] != 0: p.lost = true
g.gameMap[p.y * mapSize + p.x] = p.color
proc mainLoop(g: Game, peer: Connection) {.async.} =
while not (g.localPlayer.lost or g.remotePlayer.lost):
if g.tickTime > 0.0:
g.tickTime = 0
g.tickFinished = newFuture[int]()
# Wait for a choosen direction
let dir = await g.tickFinished
# Send it
await peer.writeLp(toBytes(uint32(dir)))
# Get the one from the peer
g.remotePlayer.currentDir = int uint32.fromBytes(await peer.readLp(8))
# Tick the players & restart
g.tick(g.remotePlayer)
g.tick(g.localPlayer)
## We'll draw the map & put some texts when necessary:
proc draw(g: Game) =
for pos, color in g.gameMap:
setColor(color)
boxFill(pos mod 32 * 4, pos div 32 * 4, 4, 4)
let text = if not(g.peerFound.finished()): "Matchmaking.."
elif g.tickTime < -1.5: "Welcome to Etron"
elif g.tickTime < 0.0: "- " & $(int(abs(g.tickTime) / 0.5) + 1) & " -"
elif g.remotePlayer.lost and g.localPlayer.lost: "DEUCE"
elif g.localPlayer.lost: "YOU LOOSE"
elif g.remotePlayer.lost: "YOU WON"
else: ""
printc(text, screenWidth div 2, screenHeight div 2)
## ## Matchmaking
## To find an opponent, we will broadcast our address on a
## GossipSub topic, and wait for someone to connect to us.
## We will also listen to that topic, and connect to anyone
## broadcasting his address.
##
## If we are looking for a game, we'll send `ok` to let the
## peer know that we are available, check that he is also available,
## and launch the game.
proc new(T: typedesc[GameProto], g: Game): T =
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
defer: await conn.closeWithEof()
if g.peerFound.finished or g.hasCandidate:
await conn.close()
return
g.hasCandidate = true
await conn.writeLp("ok")
if "ok" != string.fromBytes(await conn.readLp(1024)):
g.hasCandidate = false
return
g.peerFound.complete(conn)
# The handler of a protocol must wait for the stream to
# be finished before returning
await conn.join()
return T.new(codecs = @["/tron/1.0.0"], handler = handle)
proc networking(g: Game) {.async.} =
# Create our switch, similar to the GossipSub example and
# the Discovery examples combined
let
rdv = RendezVous.new()
switch = SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withYamux()
.withNoise()
.withRendezVous(rdv)
.build()
dm = DiscoveryManager()
gameProto = GameProto.new(g)
gossip = GossipSub.init(
switch = switch,
triggerSelf = false)
dm.add(RendezVousInterface.new(rdv))
switch.mount(gossip)
switch.mount(gameProto)
gossip.subscribe(
"/tron/matchmaking",
proc (topic: string, data: seq[byte]) {.async.} =
# If we are still looking for an opponent,
# try to match anyone broadcasting it's address
if g.peerFound.finished or g.hasCandidate: return
g.hasCandidate = true
try:
let
(peerId, multiAddress) = parseFullAddress(data).tryGet()
stream = await switch.dial(peerId, @[multiAddress], gameProto.codec)
await stream.writeLp("ok")
if (await stream.readLp(10)) != "ok".toBytes:
g.hasCandidate = false
return
g.peerFound.complete(stream)
# We are "player 2"
swap(g.localPlayer, g.remotePlayer)
except CatchableError as exc:
discard
)
await switch.start()
defer: await switch.stop()
# As explained in the last tutorial, we need a bootnode to be able
# to find peers. We could use any libp2p running rendezvous (or any
# node running tron). We will take it's MultiAddress from the command
# line parameters
if paramCount() > 0:
let (peerId, multiAddress) = paramStr(1).parseFullAddress().tryGet()
await switch.connect(peerId, @[multiAddress])
else:
echo "No bootnode provided, listening on: ", switch.peerInfo.fullAddrs.tryGet()
# Discover peers from the bootnode, and connect to them
dm.advertise(RdvNamespace("tron"))
let discoveryQuery = dm.request(RdvNamespace("tron"))
discoveryQuery.forEach:
try:
await switch.connect(peer[PeerId], peer.getAll(MultiAddress))
except CatchableError as exc:
echo "Failed to dial a peer: ", exc.msg
# We will try to publish our address multiple times, in case
# it takes time to establish connections with other GossipSub peers
var published = false
while not published:
await sleepAsync(500.milliseconds)
for fullAddr in switch.peerInfo.fullAddrs.tryGet():
if (await gossip.publish("/tron/matchmaking", fullAddr.bytes)) == 0:
published = false
break
published = true
discoveryQuery.stop()
# We now wait for someone to connect to us (or for us to connect to someone)
let peerConn = await g.peerFound
defer: await peerConn.closeWithEof()
await g.mainLoop(peerConn)
let
game = Game.new()
netFut = networking(game)
nico.init("Status", "Tron")
nico.createWindow("Tron", mapSize * 4, mapSize * 4, 4, false)
nico.run(proc = discard, proc(dt: float32) = game.update(dt), proc = game.draw())
waitFor(netFut.cancelAndWait())
## And that's it! If you want to run this code locally, the simplest way is to use the
## first node as a boot node for the second one. But you can also use any rendezvous node

2
libp2p.nim
View File

@ -17,7 +17,7 @@ when defined(nimdoc):
## stay backward compatible during the Major version, whereas private ones can
## change at each new Minor version.
##
## If you're new to nim-libp2p, you can find a tutorial `here<https://github.com/status-im/nim-libp2p/blob/master/examples/tutorial_1_connect.md>`_
## If you're new to nim-libp2p, you can find a tutorial `here<https://status-im.github.io/nim-libp2p/docs/tutorial_1_connect/>`_
## that can help you get started.
# Import stuff for doc

88
libp2p.nimble
View File

@ -1,7 +1,7 @@
mode = ScriptMode.Verbose
packageName = "libp2p"
version = "0.0.2"
version = "1.0.0"
author = "Status Research & Development GmbH"
description = "LibP2P implementation"
license = "MIT"
@ -9,7 +9,7 @@ skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
requires "nim >= 1.2.0",
"nimcrypto >= 0.4.1",
"dnsclient >= 0.1.2",
"dnsclient >= 0.3.0 & < 0.4.0",
"bearssl >= 0.1.4",
"chronicles >= 0.10.2",
"chronos >= 3.0.6",
@ -17,46 +17,36 @@ requires "nim >= 1.2.0",
"metrics",
"secp256k1",
"stew#head",
"websock"
const styleCheckStyle =
if (NimMajor, NimMinor) < (1, 6):
"hint"
else:
"error"
const nimflags =
"--verbosity:0 --hints:off " &
"--warning[CaseTransition]:off --warning[ObservableStores]:off " &
"--warning[LockLevel]:off " &
"-d:chronosStrictException " &
"--styleCheck:usages --styleCheck:" & styleCheckStyle & " "
"websock",
"unittest2 >= 0.0.5 & < 0.1.0"
import hashes
proc runTest(filename: string, verify: bool = true, sign: bool = true,
moreoptions: string = "") =
var excstr = "nim c --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics -d:libp2p_mplex_metrics "
var excstr = "nim c --skipParentCfg --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics -d:libp2p_mplex_metrics "
excstr.add(" -d:chronicles_sinks=textlines[stdout],json[dynamic] -d:chronicles_log_level=TRACE ")
excstr.add(" -d:chronicles_runtime_filtering=TRUE ")
excstr.add(" " & getEnv("NIMFLAGS") & " ")
excstr.add(" " & nimflags & " ")
excstr.add(" --verbosity:0 --hints:off ")
excstr.add(" -d:libp2p_pubsub_sign=" & $sign)
excstr.add(" -d:libp2p_pubsub_verify=" & $verify)
excstr.add(" " & moreoptions & " ")
if getEnv("CICOV").len > 0:
excstr &= " --nimcache:nimcache/" & filename & "-" & $excstr.hash
exec excstr & " -r " & " tests/" & filename
rmFile "tests/" & filename.toExe
proc buildSample(filename: string, run = false) =
var excstr = "nim c --opt:speed --threads:on -d:debug "
excstr.add(" " & nimflags & " ")
proc buildSample(filename: string, run = false, extraFlags = "") =
var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. " & extraFlags
excstr.add(" examples/" & filename)
exec excstr
if run:
exec "./examples/" & filename.toExe
rmFile "examples/" & filename.toExe
proc buildTutorial(filename: string) =
discard gorge "cat " & filename & " | nim c -r --hints:off tools/markdown_runner.nim | " &
" nim " & nimflags & " c -"
proc tutorialToMd(filename: string) =
let markdown = gorge "cat " & filename & " | nim c -r --verbosity:0 --hints:off tools/markdown_builder.nim "
writeFile(filename.replace(".nim", ".md"), markdown)
task testnative, "Runs libp2p native tests":
runTest("testnative")
@ -101,29 +91,31 @@ task test_slim, "Runs the (slimmed down) test suite":
exec "nimble testfilter"
exec "nimble examples_build"
task website, "Build the website":
tutorialToMd("examples/tutorial_1_connect.nim")
tutorialToMd("examples/tutorial_2_customproto.nim")
tutorialToMd("examples/tutorial_3_protobuf.nim")
tutorialToMd("examples/tutorial_4_gossipsub.nim")
tutorialToMd("examples/tutorial_5_discovery.nim")
tutorialToMd("examples/tutorial_6_game.nim")
tutorialToMd("examples/circuitrelay.nim")
exec "mkdocs build"
task examples_build, "Build the samples":
buildSample("directchat")
buildSample("helloworld", true)
buildSample("circuitrelay", true)
buildTutorial("examples/tutorial_1_connect.md")
buildTutorial("examples/tutorial_2_customproto.md")
proc tutorialToHtml(source, output: string) =
var html = gorge("./nimbledeps/bin/markdown < " & source)
html &= """
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/water.css@2/out/water.css">
<link rel="stylesheet" href="https://unpkg.com/@highlightjs/cdn-assets@11.5.1/styles/default.min.css">
<script src="https://unpkg.com/@highlightjs/cdn-assets@11.5.1/highlight.min.js"></script>
<script src="https://unpkg.com/@highlightjs/cdn-assets@11.5.1/languages/nim.min.js"></script>
<script>hljs.highlightAll();</script>
"""
writeFile(output, html)
task markdown_to_html, "Build the tutorials HTML":
exec "nimble install -y markdown"
tutorialToHtml("examples/tutorial_1_connect.md", "tuto1.html")
tutorialToHtml("examples/tutorial_2_customproto.md", "tuto2.html")
buildSample("tutorial_1_connect", true)
buildSample("tutorial_2_customproto", true)
if (NimMajor, NimMinor) > (1, 2):
# These tutorials relies on post 1.4 exception tracking
buildSample("tutorial_3_protobuf", true)
buildSample("tutorial_4_gossipsub", true)
buildSample("tutorial_5_discovery", true)
# Nico doesn't work in 1.2
exec "nimble install -y nimpng@#HEAD" # this is to fix broken build on 1.7.3, remove it when nimpng version 0.3.2 or later is released
exec "nimble install -y nico"
buildSample("tutorial_6_game", false, "--styleCheck:off")
# pin system
# while nimble lockfile
@ -148,9 +140,13 @@ task install_pinned, "Reads the lockfile":
# Remove the automatically installed deps
# (inefficient you say?)
let allowedDirectories = toInstall.mapIt(it[0] & "-" & it[1].split('@')[1])
for dependency in listDirs("nimbledeps/pkgs"):
if dependency.extractFilename notin allowedDirectories:
let nimblePkgs =
if system.dirExists("nimbledeps/pkgs"): "nimbledeps/pkgs"
else: "nimbledeps/pkgs2"
for dependency in listDirs(nimblePkgs):
let filename = dependency.extractFilename
if toInstall.anyIt(filename.startsWith(it[0]) and
filename.endsWith(it[1].split('#')[^1])) == false:
rmDir(dependency)
task unpin, "Restore global package use":

30
libp2p/builders.nim
View File

@ -26,8 +26,8 @@ import
switch, peerid, peerinfo, stream/connection, multiaddress,
crypto/crypto, transports/[transport, tcptransport],
muxers/[muxer, mplex/mplex, yamux/yamux],
protocols/[identify, secure/secure, secure/noise],
protocols/relay/[relay, client, rtransport],
protocols/[identify, secure/secure, secure/noise, rendezvous],
protocols/connectivity/[autonat, relay/relay, relay/client, relay/rtransport],
connmanager, upgrademngrs/muxedupgrade,
nameresolving/nameresolver,
errors, utility
@ -58,7 +58,10 @@ type
agentVersion: string
nameResolver: NameResolver
peerStoreCapacity: Option[int]
autonat: bool
circuitRelay: Relay
rdv: RendezVous
services: seq[Service]
proc new*(T: type[SwitchBuilder]): T {.public.} =
## Creates a SwitchBuilder
@ -185,10 +188,22 @@ proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuil
b.nameResolver = nameResolver
b
proc withAutonat*(b: SwitchBuilder): SwitchBuilder =
b.autonat = true
b
proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder =
b.circuitRelay = r
b
proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous = RendezVous.new()): SwitchBuilder =
b.rdv = rdv
b
proc withServices*(b: SwitchBuilder, services: seq[Service]): SwitchBuilder =
b.services = services
b
proc build*(b: SwitchBuilder): Switch
{.raises: [Defect, LPError], public.} =
@ -244,7 +259,12 @@ proc build*(b: SwitchBuilder): Switch
connManager = connManager,
ms = ms,
nameResolver = b.nameResolver,
peerStore = peerStore)
peerStore = peerStore,
services = b.services)
if b.autonat:
let autonat = Autonat.new(switch)
switch.mount(autonat)
if not isNil(b.circuitRelay):
if b.circuitRelay of RelayClient:
@ -252,6 +272,10 @@ proc build*(b: SwitchBuilder): Switch
b.circuitRelay.setup(switch)
switch.mount(b.circuitRelay)
if not isNil(b.rdv):
b.rdv.setup(switch)
switch.mount(b.rdv)
return switch
proc newStandardSwitch*(

8
libp2p/connmanager.nim
View File

@ -110,6 +110,13 @@ proc new*(C: type ConnManager,
proc connCount*(c: ConnManager, peerId: PeerId): int =
c.conns.getOrDefault(peerId).len
proc connectedPeers*(c: ConnManager, dir: Direction): seq[PeerId] =
var peers = newSeq[PeerId]()
for peerId, conns in c.conns:
if conns.anyIt(it.dir == dir):
peers.add(peerId)
return peers
proc addConnEventHandler*(c: ConnManager,
handler: ConnEventHandler,
kind: ConnEventKind) =
@ -537,3 +544,4 @@ proc close*(c: ConnManager) {.async.} =
await conn.close()
trace "Closed ConnManager"

3
libp2p/crypto/curve25519.nim
View File

@ -31,7 +31,6 @@ const
type
Curve25519* = object
Curve25519Key* = array[Curve25519KeySize, byte]
pcuchar = ptr char
Curve25519Error* = enum
Curver25519GenError
@ -77,7 +76,7 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
addr rpoint[0],
Curve25519KeySize,
EC_curve25519)
assert size == Curve25519KeySize
proc public*(private: Curve25519Key): Curve25519Key =

32
libp2p/crypto/minasn1.nim
View File

@ -528,8 +528,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass,
index: ttag, offset: int(ab.offset),
length: 1)
shallowCopy(field.buffer, ab.buffer)
length: 1, buffer: ab.buffer)
field.vbool = (b == 0xFF'u8)
ab.offset += 1
return ok(field)
@ -554,8 +553,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
# Negative or Positive integer
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
length: int(length), buffer: ab.buffer)
if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8:
# Negative integer
if length <= 8:
@ -579,16 +577,15 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
# Zero value integer
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length), vint: 0'u64)
shallowCopy(field.buffer, ab.buffer)
length: int(length), vint: 0'u64,
buffer: ab.buffer)
ab.offset += int(length)
return ok(field)
else:
# Positive integer with leading zero
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset) + 1,
length: int(length) - 1)
shallowCopy(field.buffer, ab.buffer)
length: int(length) - 1, buffer: ab.buffer)
if length <= 9:
for i in 1 ..< int(length):
field.vint = (field.vint shl 8) or
@ -610,8 +607,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
# Zero-length BIT STRING.
field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
index: ttag, offset: int(ab.offset + 1),
length: 0, ubits: 0)
shallowCopy(field.buffer, ab.buffer)
length: 0, ubits: 0, buffer: ab.buffer)
ab.offset += int(length)
return ok(field)
@ -631,8 +627,8 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
index: ttag, offset: int(ab.offset + 1),
length: int(length - 1), ubits: int(unused))
shallowCopy(field.buffer, ab.buffer)
length: int(length - 1), ubits: int(unused),
buffer: ab.buffer)
ab.offset += int(length)
return ok(field)
@ -643,8 +639,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
length: int(length), buffer: ab.buffer)
ab.offset += int(length)
return ok(field)
@ -654,8 +649,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag,
offset: int(ab.offset), length: 0)
shallowCopy(field.buffer, ab.buffer)
offset: int(ab.offset), length: 0, buffer: ab.buffer)
ab.offset += int(length)
return ok(field)
@ -666,8 +660,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
length: int(length), buffer: ab.buffer)
ab.offset += int(length)
return ok(field)
@ -678,8 +671,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
length: int(length), buffer: ab.buffer)
ab.offset += int(length)
return ok(field)

2
libp2p/daemon/daemonapi.nim
View File

@ -492,7 +492,7 @@ proc recvMessage(conn: StreamTransport): Future[seq[byte]] {.async.} =
res = PB.getUVarint(buffer.toOpenArray(0, i), length, size)
if res.isOk():
break
if res.isErr() or size > MaxMessageSize:
if res.isErr() or size > 1'u shl 22:
buffer.setLen(0)
result = buffer
return

17
libp2p/dial.nim
View File

@ -13,10 +13,13 @@ else:
{.push raises: [].}
import chronos
import stew/results
import peerid,
stream/connection,
transports/transport
export results
type
Dial* = ref object of RootObj
@ -31,6 +34,14 @@ method connect*(
doAssert(false, "Not implemented!")
method connect*(
self: Dial,
address: MultiAddress,
allowUnknownPeerId = false): Future[PeerId] {.async, base.} =
## Connects to a peer and retrieve its PeerId
doAssert(false, "Not implemented!")
method dial*(
self: Dial,
peerId: PeerId,
@ -58,3 +69,9 @@ method addTransport*(
self: Dial,
transport: Transport) {.base.} =
doAssert(false, "Not implemented!")
method tryDial*(
self: Dial,
peerId: PeerId,
addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async, base.} =
doAssert(false, "Not implemented!")

198
libp2p/dialer.nim
View File

@ -7,8 +7,9 @@
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/[sugar, tables]
import std/[sugar, tables, sequtils]
import stew/results
import pkg/[chronos,
chronicles,
metrics]
@ -16,14 +17,16 @@ import pkg/[chronos,
import dial,
peerid,
peerinfo,
multicodec,
multistream,
connmanager,
stream/connection,
transports/transport,
nameresolving/nameresolver,
upgrademngrs/upgrade,
errors
export dial, errors
export dial, errors, results
logScope:
topics = "libp2p dialer"
@ -46,74 +49,122 @@ type
proc dialAndUpgrade(
self: Dialer,
peerId: PeerId,
peerId: Opt[PeerId],
hostname: string,
address: MultiAddress):
Future[Connection] {.async.} =
for transport in self.transports: # for each transport
if transport.handles(address): # check if it can dial it
trace "Dialing address", address, peerId, hostname
let dialed =
try:
libp2p_total_dial_attempts.inc()
await transport.dial(hostname, address)
except CancelledError as exc:
debug "Dialing canceled", msg = exc.msg, peerId
raise exc
except CatchableError as exc:
debug "Dialing failed", msg = exc.msg, peerId
libp2p_failed_dials.inc()
return nil # Try the next address
# also keep track of the connection's bottom unsafe transport direction
# required by gossipsub scoring
dialed.transportDir = Direction.Out
libp2p_successful_dials.inc()
let conn =
try:
await transport.upgradeOutgoing(dialed, peerId)
except CatchableError as exc:
# If we failed to establish the connection through one transport,
# we won't succeeded through another - no use in trying again
await dialed.close()
debug "Upgrade failed", msg = exc.msg, peerId
if exc isnot CancelledError:
libp2p_failed_upgrades_outgoing.inc()
# Try other address
return nil
doAssert not isNil(conn), "connection died after upgradeOutgoing"
debug "Dial successful", conn, peerId = conn.peerId
return conn
return nil
proc expandDnsAddr(
self: Dialer,
peerId: Opt[PeerId],
address: MultiAddress): Future[seq[(MultiAddress, Opt[PeerId])]] {.async.} =
if not DNSADDR.matchPartial(address): return @[(address, peerId)]
if isNil(self.nameResolver):
info "Can't resolve DNSADDR without NameResolver", ma=address
return @[]
let
toResolve =
if peerId.isSome:
address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet()
else:
address
resolved = await self.nameResolver.resolveDnsAddr(toResolve)
for resolvedAddress in resolved:
let lastPart = resolvedAddress[^1].tryGet()
if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")):
let
peerIdBytes = lastPart.protoArgument().tryGet()
addrPeerId = PeerId.init(peerIdBytes).tryGet()
result.add((resolvedAddress[0..^2].tryGet(), Opt.some(addrPeerId)))
else:
result.add((resolvedAddress, peerId))
proc dialAndUpgrade(
self: Dialer,
peerId: Opt[PeerId],
addrs: seq[MultiAddress]):
Future[Connection] {.async.} =
debug "Dialing peer", peerId
for address in addrs: # for each address
let
hostname = address.getHostname()
resolvedAddresses =
if isNil(self.nameResolver): @[address]
else: await self.nameResolver.resolveMAddress(address)
for rawAddress in addrs:
# resolve potential dnsaddr
let addresses = await self.expandDnsAddr(peerId, rawAddress)
for a in resolvedAddresses: # for each resolved address
for transport in self.transports: # for each transport
if transport.handles(a): # check if it can dial it
trace "Dialing address", address = $a, peerId, hostname
let dialed = try:
libp2p_total_dial_attempts.inc()
await transport.dial(hostname, a)
except CancelledError as exc:
debug "Dialing canceled", msg = exc.msg, peerId
raise exc
except CatchableError as exc:
debug "Dialing failed", msg = exc.msg, peerId
libp2p_failed_dials.inc()
continue # Try the next address
for (expandedAddress, addrPeerId) in addresses:
# DNS resolution
let
hostname = expandedAddress.getHostname()
resolvedAddresses =
if isNil(self.nameResolver): @[expandedAddress]
else: await self.nameResolver.resolveMAddress(expandedAddress)
# make sure to assign the peer to the connection
dialed.peerId = peerId
# also keep track of the connection's bottom unsafe transport direction
# required by gossipsub scoring
dialed.transportDir = Direction.Out
libp2p_successful_dials.inc()
let conn = try:
await transport.upgradeOutgoing(dialed)
except CatchableError as exc:
# If we failed to establish the connection through one transport,
# we won't succeeded through another - no use in trying again
# TODO we should try another address though
await dialed.close()
debug "Upgrade failed", msg = exc.msg, peerId
if exc isnot CancelledError:
libp2p_failed_upgrades_outgoing.inc()
raise exc
doAssert not isNil(conn), "connection died after upgradeOutgoing"
debug "Dial successful", conn, peerId = conn.peerId
return conn
for resolvedAddress in resolvedAddresses:
result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress)
if not isNil(result):
return result
proc internalConnect(
self: Dialer,
peerId: PeerId,
peerId: Opt[PeerId],
addrs: seq[MultiAddress],
forceDial: bool):
Future[Connection] {.async.} =
if self.localPeerId == peerId:
if Opt.some(self.localPeerId) == peerId:
raise newException(CatchableError, "can't dial self!")
# Ensure there's only one in-flight attempt per peer
let lock = self.dialLock.mgetOrPut(peerId, newAsyncLock())
let lock = self.dialLock.mgetOrPut(peerId.get(default(PeerId)), newAsyncLock())
try:
await lock.acquire()
# Check if we have a connection already and try to reuse it
var conn = self.connManager.selectConn(peerId)
var conn =
if peerId.isSome: self.connManager.selectConn(peerId.get())
else: nil
if conn != nil:
if conn.atEof or conn.closed:
# This connection should already have been removed from the connection
@ -164,7 +215,27 @@ method connect*(
if self.connManager.connCount(peerId) > 0:
return
discard await self.internalConnect(peerId, addrs, forceDial)
discard await self.internalConnect(Opt.some(peerId), addrs, forceDial)
method connect*(
self: Dialer,
address: MultiAddress,
allowUnknownPeerId = false): Future[PeerId] {.async.} =
## Connects to a peer and retrieve its PeerId
let fullAddress = parseFullAddress(address)
if fullAddress.isOk:
return (await self.internalConnect(
Opt.some(fullAddress.get()[0]),
@[fullAddress.get()[1]],
false)).peerId
else:
if allowUnknownPeerId == false:
raise newException(DialFailedError, "Address without PeerID and unknown peer id disabled!")
return (await self.internalConnect(
Opt.none(PeerId),
@[address],
false)).peerId
proc negotiateStream(
self: Dialer,
@ -178,6 +249,27 @@ proc negotiateStream(
return conn
method tryDial*(
self: Dialer,
peerId: PeerId,
addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async.} =
## Create a protocol stream in order to check
## if a connection is possible.
## Doesn't use the Connection Manager to save it.
##
trace "Check if it can dial", peerId, addrs
try:
let conn = await self.dialAndUpgrade(Opt.some(peerId), addrs)
if conn.isNil():
raise newException(DialFailedError, "No valid multiaddress")
await conn.close()
return conn.observedAddr
except CancelledError as exc:
raise exc
except CatchableError as exc:
raise newException(DialFailedError, exc.msg)
method dial*(
self: Dialer,
peerId: PeerId,
@ -216,7 +308,7 @@ method dial*(
try:
trace "Dialing (new)", peerId, protos
conn = await self.internalConnect(peerId, addrs, forceDial)
conn = await self.internalConnect(Opt.some(peerId), addrs, forceDial)
trace "Opening stream", conn
stream = await self.connManager.getStream(conn)

182
libp2p/discovery/discoverymngr.nim Normal file
View File

@ -0,0 +1,182 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import std/sequtils
import chronos, chronicles, stew/results
import ../errors
type
BaseAttr = ref object of RootObj
comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [Defect].}
Attribute[T] = ref object of BaseAttr
value: T
PeerAttributes* = object
attributes: seq[BaseAttr]
DiscoveryService* = distinct string
proc `==`*(a, b: DiscoveryService): bool {.borrow.}
proc ofType*[T](f: BaseAttr, _: type[T]): bool =
return f of Attribute[T]
proc to*[T](f: BaseAttr, _: type[T]): T =
Attribute[T](f).value
proc add*[T](pa: var PeerAttributes,
value: T) =
pa.attributes.add(Attribute[T](
value: value,
comparator: proc(f: BaseAttr, c: BaseAttr): bool =
f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T)
)
)
iterator items*(pa: PeerAttributes): BaseAttr =
for f in pa.attributes:
yield f
proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] =
for f in pa.attributes:
if f.ofType(T):
result.add(f.to(T))
proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] =
for f in pa.attributes:
if f.ofType(T):
return Opt.some(f.to(T))
Opt.none(T)
proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [Defect, KeyError].} =
pa{T}.valueOr: raise newException(KeyError, "Attritute not found")
proc match*(pa, candidate: PeerAttributes): bool =
for f in pa.attributes:
block oneAttribute:
for field in candidate.attributes:
if field.comparator(field, f):
break oneAttribute
return false
return true
type
PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [Defect], gcsafe.}
DiscoveryInterface* = ref object of RootObj
onPeerFound*: PeerFoundCallback
toAdvertise*: PeerAttributes
advertisementUpdated*: AsyncEvent
advertiseLoop*: Future[void]
method request*(self: DiscoveryInterface, pa: PeerAttributes) {.async, base.} =
doAssert(false, "Not implemented!")
method advertise*(self: DiscoveryInterface) {.async, base.} =
doAssert(false, "Not implemented!")
type
DiscoveryError* = object of LPError
DiscoveryFinished* = object of LPError
DiscoveryQuery* = ref object
attr: PeerAttributes
peers: AsyncQueue[PeerAttributes]
finished: bool
futs: seq[Future[void]]
DiscoveryManager* = ref object
interfaces: seq[DiscoveryInterface]
queries: seq[DiscoveryQuery]
proc add*(dm: DiscoveryManager, di: DiscoveryInterface) =
dm.interfaces &= di
di.onPeerFound = proc (pa: PeerAttributes) =
for query in dm.queries:
if query.attr.match(pa):
try:
query.peers.putNoWait(pa)
except AsyncQueueFullError as exc:
debug "Cannot push discovered peer to queue"
proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery =
var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]())
for i in dm.interfaces:
query.futs.add(i.request(pa))
dm.queries.add(query)
dm.queries.keepItIf(it.futs.anyIt(not it.finished()))
return query
proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery =
var pa: PeerAttributes
pa.add(value)
return dm.request(pa)
proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) =
for i in dm.interfaces:
i.toAdvertise = pa
if i.advertiseLoop.isNil:
i.advertisementUpdated = newAsyncEvent()
i.advertiseLoop = i.advertise()
else:
i.advertisementUpdated.fire()
proc advertise*[T](dm: DiscoveryManager, value: T) =
var pa: PeerAttributes
pa.add(value)
dm.advertise(pa)
template forEach*(query: DiscoveryQuery, code: untyped) =
## Will execute `code` for each discovered peer. The
## peer attritubtes are available through the variable
## `peer`
proc forEachInternal(q: DiscoveryQuery) {.async.} =
while true:
let peer {.inject.} =
try: await q.getPeer()
except DiscoveryFinished: return
code
asyncSpawn forEachInternal(query)
proc stop*(query: DiscoveryQuery) =
query.finished = true
for r in query.futs:
if not r.finished(): r.cancel()
proc stop*(dm: DiscoveryManager) =
for q in dm.queries:
q.stop()
for i in dm.interfaces:
if isNil(i.advertiseLoop): continue
i.advertiseLoop.cancel()
proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} =
let getter = query.peers.popFirst()
try:
await getter or allFinished(query.futs)
except CancelledError as exc:
getter.cancel()
raise exc
if not finished(getter):
if query.finished:
raise newException(DiscoveryFinished, "Discovery query stopped")
# discovery loops only finish when they don't handle the query
raise newException(DiscoveryError, "Unable to find any peer matching this request")
return await getter

77
libp2p/discovery/rendezvousinterface.nim Normal file
View File

@ -0,0 +1,77 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import sequtils
import chronos
import ./discoverymngr,
../protocols/rendezvous,
../peerid
type
RendezVousInterface* = ref object of DiscoveryInterface
rdv*: RendezVous
timeToRequest: Duration
timeToAdvertise: Duration
RdvNamespace* = distinct string
proc `==`*(a, b: RdvNamespace): bool {.borrow.}
method request*(self: RendezVousInterface, pa: PeerAttributes) {.async.} =
var namespace = ""
for attr in pa:
if attr.ofType(RdvNamespace):
namespace = string attr.to(RdvNamespace)
elif attr.ofType(DiscoveryService):
namespace = string attr.to(DiscoveryService)
elif attr.ofType(PeerId):
namespace = $attr.to(PeerId)
else:
# unhandled type
return
while true:
for pr in await self.rdv.request(namespace):
var peer: PeerAttributes
peer.add(pr.peerId)
for address in pr.addresses:
peer.add(address.address)
peer.add(DiscoveryService(namespace))
peer.add(RdvNamespace(namespace))
self.onPeerFound(peer)
await sleepAsync(self.timeToRequest)
method advertise*(self: RendezVousInterface) {.async.} =
while true:
var toAdvertise: seq[string]
for attr in self.toAdvertise:
if attr.ofType(RdvNamespace):
toAdvertise.add string attr.to(RdvNamespace)
elif attr.ofType(DiscoveryService):
toAdvertise.add string attr.to(DiscoveryService)
elif attr.ofType(PeerId):
toAdvertise.add $attr.to(PeerId)
self.advertisementUpdated.clear()
for toAdv in toAdvertise:
await self.rdv.advertise(toAdv, self.timeToAdvertise)
await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait()
proc new*(T: typedesc[RendezVousInterface],
rdv: RendezVous,
ttr: Duration = 1.minutes,
tta: Duration = MinimumDuration): RendezVousInterface =
T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta)

113
libp2p/multiaddress.nim
View File

@ -222,6 +222,40 @@ proc onionVB(vb: var VBuffer): bool =
if vb.readArray(buf) == 12:
result = true
proc onion3StB(s: string, vb: var VBuffer): bool =
try:
var parts = s.split(':')
if len(parts) != 2:
return false
if len(parts[0]) != 56:
return false
var address = Base32Lower.decode(parts[0].toLowerAscii())
var nport = parseInt(parts[1])
if (nport > 0 and nport < 65536) and len(address) == 35:
address.setLen(37)
address[35] = cast[byte]((nport shr 8) and 0xFF)
address[36] = cast[byte](nport and 0xFF)
vb.writeArray(address)
result = true
except:
discard
proc onion3BtS(vb: var VBuffer, s: var string): bool =
## ONION address bufferToString() implementation.
var buf: array[37, byte]
if vb.readArray(buf) == 37:
var nport = (cast[uint16](buf[35]) shl 8) or cast[uint16](buf[36])
s = Base32Lower.encode(buf.toOpenArray(0, 34))
s.add(":")
s.add($nport)
result = true
proc onion3VB(vb: var VBuffer): bool =
## ONION address validateBuffer() implementation.
var buf: array[37, byte]
if vb.readArray(buf) == 37:
result = true
proc unixStB(s: string, vb: var VBuffer): bool =
## Unix socket name stringToBuffer() implementation.
if len(s) > 0:
@ -310,6 +344,11 @@ const
bufferToString: onionBtS,
validateBuffer: onionVB
)
TranscoderOnion3* = Transcoder(
stringToBuffer: onion3StB,
bufferToString: onion3BtS,
validateBuffer: onion3VB
)
TranscoderDNS* = Transcoder(
stringToBuffer: dnsStB,
bufferToString: dnsBtS,
@ -363,6 +402,10 @@ const
mcodec: multiCodec("onion"), kind: Fixed, size: 10,
coder: TranscoderOnion
),
MAProtocol(
mcodec: multiCodec("onion3"), kind: Fixed, size: 37,
coder: TranscoderOnion3
),
MAProtocol(
mcodec: multiCodec("ws"), kind: Marker, size: 0
),
@ -427,6 +470,8 @@ const
WS* = mapAnd(TCP, mapEq("ws"))
WSS* = mapAnd(TCP, mapEq("wss"))
WebSockets* = mapOr(WS, WSS)
Onion3* = mapEq("onion3")
TcpOnion3* = mapAnd(TCP, Onion3)
Unreliable* = mapOr(UDP)
@ -473,15 +518,10 @@ proc trimRight(s: string, ch: char): string =
break
result = s[0..(s.high - m)]
proc shcopy*(m1: var MultiAddress, m2: MultiAddress) =
shallowCopy(m1.data.buffer, m2.data.buffer)
m1.data.offset = m2.data.offset
proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
## Returns MultiAddress ``ma`` protocol code.
var header: uint64
var vb: MultiAddress
shcopy(vb, ma)
var vb = ma
if vb.data.readVarint(header) == -1:
err("multiaddress: Malformed binary address!")
else:
@ -494,8 +534,7 @@ proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
proc protoName*(ma: MultiAddress): MaResult[string] =
## Returns MultiAddress ``ma`` protocol name.
var header: uint64
var vb: MultiAddress
shcopy(vb, ma)
var vb = ma
if vb.data.readVarint(header) == -1:
err("multiaddress: Malformed binary address!")
else:
@ -512,9 +551,8 @@ proc protoArgument*(ma: MultiAddress,
## If current MultiAddress do not have argument value, then result will be
## ``0``.
var header: uint64
var vb: MultiAddress
var vb = ma
var buffer: seq[byte]
shcopy(vb, ma)
if vb.data.readVarint(header) == -1:
err("multiaddress: Malformed binary address!")
else:
@ -530,7 +568,7 @@ proc protoArgument*(ma: MultiAddress,
err("multiaddress: Decoding protocol error")
else:
ok(res)
elif proto.kind in {Length, Path}:
elif proto.kind in {MAKind.Length, Path}:
if vb.data.readSeq(buffer) == -1:
err("multiaddress: Decoding protocol error")
else:
@ -551,6 +589,13 @@ proc protoAddress*(ma: MultiAddress): MaResult[seq[byte]] =
buffer.setLen(res)
ok(buffer)
proc protoArgument*(ma: MultiAddress): MaResult[seq[byte]] =
## Returns MultiAddress ``ma`` protocol address binary blob.
##
## If current MultiAddress do not have argument value, then result array will
## be empty.
ma.protoAddress()
proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
var header: uint64
var data = newSeq[byte]()
@ -558,6 +603,9 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
var vb = ma
var res: MultiAddress
res.data = initVBuffer()
if index < 0: return err("multiaddress: negative index gived to getPart")
while offset <= index:
if vb.data.readVarint(header) == -1:
return err("multiaddress: Malformed binary address!")
@ -575,7 +623,7 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
res.data.writeVarint(header)
res.data.writeArray(data)
res.data.finish()
elif proto.kind in {Length, Path}:
elif proto.kind in {MAKind.Length, Path}:
if vb.data.readSeq(data) == -1:
return err("multiaddress: Decoding protocol error")
@ -590,9 +638,31 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
inc(offset)
ok(res)
proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} =
proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddress] =
when slice.a is BackwardsIndex or slice.b is BackwardsIndex:
let maLength = ? len(ma)
template normalizeIndex(index): int =
when index is BackwardsIndex: maLength - int(index)
else: int(index)
let
indexStart = normalizeIndex(slice.a)
indexEnd = normalizeIndex(slice.b)
var res: MultiAddress
for i in indexStart..indexEnd:
? res.append(? ma[i])
ok(res)
proc `[]`*(ma: MultiAddress, i: int | BackwardsIndex): MaResult[MultiAddress] {.inline.} =
## Returns part with index ``i`` of MultiAddress ``ma``.
ma.getPart(i)
when i is BackwardsIndex:
let maLength = ? len(ma)
ma.getPart(maLength - int(i))
else:
ma.getPart(i)
proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} =
## Returns parts with slice ``slice`` of MultiAddress ``ma``.
ma.getParts(slice)
iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
## Iterates over all addresses inside of MultiAddress ``ma``.
@ -619,7 +689,7 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
res.data.writeVarint(header)
res.data.writeArray(data)
elif proto.kind in {Length, Path}:
elif proto.kind in {MAKind.Length, Path}:
if vb.data.readSeq(data) == -1:
yield err(MaResult[MultiAddress], "Decoding protocol error")
@ -630,6 +700,13 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
res.data.finish()
yield ok(MaResult[MultiAddress], res)
proc len*(ma: MultiAddress): MaResult[int] =
var counter: int
for part in ma:
if part.isErr: return err(part.error)
counter.inc()
ok(counter)
proc contains*(ma: MultiAddress, codec: MultiCodec): MaResult[bool] {.inline.} =
## Returns ``true``, if address with MultiCodec ``codec`` present in
## MultiAddress ``ma``.
@ -710,8 +787,7 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
proc validate*(ma: MultiAddress): bool =
## Returns ``true`` if MultiAddress ``ma`` is valid.
var header: uint64
var vb: MultiAddress
shcopy(vb, ma)
var vb = ma
while true:
if vb.data.isEmpty():
break
@ -1010,6 +1086,9 @@ proc `$`*(pat: MaPattern): string =
elif pat.operator == Eq:
result = $pat.value
proc bytes*(value: MultiAddress): seq[byte] =
value.data.buffer
proc write*(pb: var ProtoBuffer, field: int, value: MultiAddress) {.inline.} =
write(pb, field, value.data.buffer)

1
libp2p/multicodec.nim
View File

@ -203,6 +203,7 @@ const MultiCodecList = [
("p2p-webrtc-star", 0x0113), # not in multicodec list
("p2p-webrtc-direct", 0x0114), # not in multicodec list
("onion", 0x01BC),
("onion3", 0x01BD),
("p2p-circuit", 0x0122),
("libp2p-peer-record", 0x0301),
("dns", 0x35),

30
libp2p/multistream.nim
View File

@ -12,7 +12,7 @@ when (NimMajor, NimMinor) < (1, 4):
else:
{.push raises: [].}
import std/[strutils, sequtils]
import std/[strutils, sequtils, tables]
import chronos, chronicles, stew/byteutils
import stream/connection,
protocols/protocol
@ -21,7 +21,7 @@ logScope:
topics = "libp2p multistream"
const
MsgSize* = 64*1024
MsgSize* = 1024
Codec* = "/multistream/1.0.0"
MSCodec* = "\x13" & Codec & "\n"
@ -33,17 +33,20 @@ type
MultiStreamError* = object of LPError
HandlerHolder* = object
HandlerHolder* = ref object
protos*: seq[string]
protocol*: LPProtocol
match*: Matcher
openedStreams: CountTable[PeerId]
MultistreamSelect* = ref object of RootObj
handlers*: seq[HandlerHolder]
codec*: string
proc new*(T: typedesc[MultistreamSelect]): T =
T(codec: MSCodec)
T(
codec: MSCodec,
)
template validateSuffix(str: string): untyped =
if str.endsWith("\n"):
@ -169,9 +172,22 @@ proc handle*(m: MultistreamSelect, conn: Connection, active: bool = false) {.asy
for h in m.handlers:
if (not isNil(h.match) and h.match(ms)) or h.protos.contains(ms):
trace "found handler", conn, protocol = ms
await conn.writeLp(ms & "\n")
conn.protocol = ms
await h.protocol.handler(conn, ms)
var protocolHolder = h
let maxIncomingStreams = protocolHolder.protocol.maxIncomingStreams
if protocolHolder.openedStreams.getOrDefault(conn.peerId) >= maxIncomingStreams:
debug "Max streams for protocol reached, blocking new stream",
conn, protocol = ms, maxIncomingStreams
return
protocolHolder.openedStreams.inc(conn.peerId)
try:
await conn.writeLp(ms & "\n")
conn.protocol = ms
await protocolHolder.protocol.handler(conn, ms)
finally:
protocolHolder.openedStreams.inc(conn.peerId, -1)
if protocolHolder.openedStreams[conn.peerId] == 0:
protocolHolder.openedStreams.del(conn.peerId)
return
debug "no handlers", conn, protocol = ms
await conn.write(Na)

21
libp2p/muxers/mplex/lpchannel.nim
View File

@ -58,6 +58,8 @@ type
initiator*: bool # initiated remotely or locally flag
isOpen*: bool # has channel been opened
closedLocal*: bool # has channel been closed locally
remoteReset*: bool # has channel been remotely reset
localReset*: bool # has channel been reset locally
msgCode*: MessageType # cached in/out message code
closeCode*: MessageType # cached in/out close code
resetCode*: MessageType # cached in/out reset code
@ -103,6 +105,7 @@ proc reset*(s: LPChannel) {.async, gcsafe.} =
s.isClosed = true
s.closedLocal = true
s.localReset = not s.remoteReset
trace "Resetting channel", s, len = s.len
@ -168,6 +171,14 @@ method readOnce*(s: LPChannel,
## channels are blocked - in particular, this means that reading from one
## channel must not be done from within a callback / read handler of another
## or the reads will lock each other.
if s.remoteReset:
raise newLPStreamResetError()
if s.localReset:
raise newLPStreamClosedError()
if s.atEof():
raise newLPStreamRemoteClosedError()
if s.conn.closed:
raise newLPStreamConnDownError()
try:
let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes)
when defined(libp2p_network_protocols_metrics):
@ -184,13 +195,17 @@ method readOnce*(s: LPChannel,
# data has been lost in s.readBuf and there's no way to gracefully recover /
# use the channel any more
await s.reset()
raise exc
raise newLPStreamConnDownError(exc)
proc prepareWrite(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
# prepareWrite is the slow path of writing a message - see conditions in
# write
if s.closedLocal or s.conn.closed:
if s.remoteReset:
raise newLPStreamResetError()
if s.closedLocal:
raise newLPStreamClosedError()
if s.conn.closed:
raise newLPStreamConnDownError()
if msg.len == 0:
return
@ -235,7 +250,7 @@ proc completeWrite(
trace "exception in lpchannel write handler", s, msg = exc.msg
await s.reset()
await s.conn.close()
raise exc
raise newLPStreamConnDownError(exc)
finally:
s.writes -= 1

1
libp2p/muxers/mplex/mplex.nim
View File

@ -183,6 +183,7 @@ method handle*(m: Mplex) {.async, gcsafe.} =
of MessageType.CloseIn, MessageType.CloseOut:
await channel.pushEof()
of MessageType.ResetIn, MessageType.ResetOut:
channel.remoteReset = true
await channel.reset()
except CancelledError:
debug "Unexpected cancellation in mplex handler", m

68
libp2p/muxers/yamux/yamux.nim
View File

@ -153,6 +153,7 @@ type
sendQueue: seq[ToSend]
recvQueue: seq[byte]
isReset: bool
remoteReset: bool
closedRemotely: Future[void]
closedLocally: bool
receivedData: AsyncEvent
@ -194,23 +195,25 @@ method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
await channel.actuallyClose()
proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} =
if not channel.isReset:
trace "Reset channel"
channel.isReset = true
for (d, s, fut) in channel.sendQueue:
fut.fail(newLPStreamEOFError())
channel.sendQueue = @[]
channel.recvQueue = @[]
channel.sendWindow = 0
if not channel.closedLocally:
if isLocal:
try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
except LPStreamEOFError as exc: discard
except LPStreamClosedError as exc: discard
await channel.close()
if not channel.closedRemotely.done():
await channel.remoteClosed()
channel.receivedData.fire()
if channel.isReset:
return
trace "Reset channel"
channel.isReset = true
channel.remoteReset = not isLocal
for (d, s, fut) in channel.sendQueue:
fut.fail(newLPStreamEOFError())
channel.sendQueue = @[]
channel.recvQueue = @[]
channel.sendWindow = 0
if not channel.closedLocally:
if isLocal:
try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
except LPStreamEOFError as exc: discard
except LPStreamClosedError as exc: discard
await channel.close()
if not channel.closedRemotely.done():
await channel.remoteClosed()
channel.receivedData.fire()
if not isLocal:
# If we reset locally, we want to flush up to a maximum of recvWindow
# bytes. We use the recvWindow in the proc cleanupChann.
@ -235,7 +238,15 @@ method readOnce*(
nbytes: int):
Future[int] {.async.} =
if channel.returnedEof: raise newLPStreamEOFError()
if channel.isReset:
raise if channel.remoteReset:
newLPStreamResetError()
elif channel.closedLocally:
newLPStreamClosedError()
else:
newLPStreamConnDownError()
if channel.returnedEof:
raise newLPStreamRemoteClosedError()
if channel.recvQueue.len == 0:
channel.receivedData.clear()
await channel.closedRemotely or channel.receivedData.wait()
@ -313,8 +324,9 @@ proc trySend(channel: YamuxChannel) {.async.} =
channel.sendWindow.dec(toSend)
try: await channel.conn.write(sendBuffer)
except CatchableError as exc:
let connDown = newLPStreamConnDownError(exc)
for fut in futures.items():
fut.fail(exc)
fut.fail(connDown)
await channel.reset()
break
for fut in futures.items():
@ -323,8 +335,11 @@ proc trySend(channel: YamuxChannel) {.async.} =
method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] =
result = newFuture[void]("Yamux Send")
if channel.remoteReset:
result.fail(newLPStreamResetError())
return result
if channel.closedLocally or channel.isReset:
result.fail(newLPStreamEOFError())
result.fail(newLPStreamClosedError())
return result
if msg.len == 0:
result.complete()
@ -396,9 +411,11 @@ method close*(m: Yamux) {.async.} =
m.isClosed = true
trace "Closing yamux"
for channel in m.channels.values:
await channel.reset()
await m.connection.write(YamuxHeader.goAway(NormalTermination))
let channels = toSeq(m.channels.values())
for channel in channels:
await channel.reset(true)
try: await m.connection.write(YamuxHeader.goAway(NormalTermination))
except CatchableError as exc: trace "failed to send goAway", msg=exc.msg
await m.connection.close()
trace "Closed yamux"
@ -453,8 +470,9 @@ method handle*(m: Yamux) {.async, gcsafe.} =
m.flushed[header.streamId].dec(int(header.length))
if m.flushed[header.streamId] < 0:
raise newException(YamuxError, "Peer exhausted the recvWindow after reset")
var buffer = newSeqUninitialized[byte](header.length)
await m.connection.readExactly(addr buffer[0], int(header.length))
if header.length > 0:
var buffer = newSeqUninitialized[byte](header.length)
await m.connection.readExactly(addr buffer[0], int(header.length))
continue
let channel = m.channels[header.streamId]

24
libp2p/nameresolving/dnsresolver.nim
View File

@ -14,7 +14,7 @@ else:
import
std/[streams, strutils, sets, sequtils],
chronos, chronicles,
chronos, chronicles, stew/byteutils,
dnsclientpkg/[protocol, types]
import
@ -76,15 +76,11 @@ proc getDnsResponse(
if not receivedDataFuture.finished:
raise newException(IOError, "DNS server timeout")
var
rawResponse = sock.getMessage()
dataStream = newStringStream()
dataStream.writeData(addr rawResponse[0], rawResponse.len)
dataStream.setPosition(0)
let rawResponse = sock.getMessage()
# parseResponse can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
return parseResponse(dataStream)
return parseResponse(string.fromBytes(rawResponse))
except CatchableError as exc: raise exc
except Exception as exc: raiseAssert exc.msg
finally:
@ -118,7 +114,14 @@ method resolveIp*(
try:
let resp = await fut
for answer in resp.answers:
resolvedAddresses.incl(answer.toString())
# toString can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
resolvedAddresses.incl(
try: answer.toString()
except CatchableError as exc: raise exc
except Exception as exc: raiseAssert exc.msg
)
except CancelledError as e:
raise e
except ValueError as e:
@ -158,6 +161,11 @@ method resolveTxt*(
self.nameServers.add(self.nameServers[0])
self.nameServers.delete(0)
continue
except Exception as e:
# toString can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
raiseAssert e.msg
debug "Failed to resolve TXT, returning empty set"
return @[]

64
libp2p/nameresolving/nameresolver.nim
View File

@ -13,7 +13,7 @@ else:
{.push raises: [].}
import std/[sugar, sets, sequtils, strutils]
import
import
chronos,
chronicles,
stew/[endians2, byteutils]
@ -22,14 +22,14 @@ import ".."/[multiaddress, multicodec]
logScope:
topics = "libp2p nameresolver"
type
type
NameResolver* = ref object of RootObj
method resolveTxt*(
self: NameResolver,
address: string): Future[seq[string]] {.async, base.} =
## Get TXT record
##
##
doAssert(false, "Not implemented!")
@ -39,16 +39,18 @@ method resolveIp*(
port: Port,
domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async, base.} =
## Resolve the specified address
##
##
doAssert(false, "Not implemented!")
proc getHostname*(ma: MultiAddress): string =
let firstPart = ($ma[0].get()).split('/')
if firstPart.len > 1: firstPart[2]
let
firstPart = ma[0].valueOr: return ""
fpSplitted = ($firstPart).split('/', 2)
if fpSplitted.len > 2: fpSplitted[2]
else: ""
proc resolveDnsAddress(
proc resolveOneAddress(
self: NameResolver,
ma: MultiAddress,
domain: Domain = Domain.AF_UNSPEC,
@ -64,29 +66,22 @@ proc resolveDnsAddress(
let
port = Port(fromBytesBE(uint16, pbuf))
resolvedAddresses = await self.resolveIp(prefix & dnsval, port, domain)
return collect(newSeqOfCap(4)):
for address in resolvedAddresses:
var createdAddress = MultiAddress.init(address).tryGet()[0].tryGet()
for part in ma:
if DNS.match(part.get()): continue
if DNS.match(part.tryGet()): continue
createdAddress &= part.tryGet()
createdAddress
func matchDnsSuffix(m1, m2: MultiAddress): MaResult[bool] =
for partMaybe in m1:
let part = ?partMaybe
if DNS.match(part): continue
let entryProt = ?m2[?part.protoCode()]
if entryProt != part:
return ok(false)
return ok(true)
proc resolveDnsAddr(
proc resolveDnsAddr*(
self: NameResolver,
ma: MultiAddress,
depth: int = 0): Future[seq[MultiAddress]]
{.async.} =
depth: int = 0): Future[seq[MultiAddress]] {.async.} =
if not DNSADDR.matchPartial(ma):
return @[ma]
trace "Resolving dnsaddr", ma
if depth > 6:
@ -104,21 +99,17 @@ proc resolveDnsAddr(
if not entry.startsWith("dnsaddr="): continue
let entryValue = MultiAddress.init(entry[8..^1]).tryGet()
if not matchDnsSuffix(ma, entryValue).tryGet(): continue
if entryValue.contains(multiCodec("p2p")).tryGet() and ma.contains(multiCodec("p2p")).tryGet():
if entryValue[multiCodec("p2p")] != ma[multiCodec("p2p")]:
continue
# The spec is not clear wheter only DNSADDR can be recursived
# or any DNS addr. Only handling DNSADDR because it's simpler
# to avoid infinite recursion
if DNSADDR.matchPartial(entryValue):
let resolved = await self.resolveDnsAddr(entryValue, depth + 1)
for r in resolved:
result.add(r)
else:
result.add(entryValue)
let resolved = await self.resolveDnsAddr(entryValue, depth + 1)
for r in resolved:
result.add(r)
if result.len == 0:
debug "Failed to resolve any DNSADDR", ma
return @[ma]
debug "Failed to resolve a DNSADDR", ma
return @[]
return result
@ -133,14 +124,15 @@ proc resolveMAddress*(
let code = address[0].get().protoCode().get()
let seq = case code:
of multiCodec("dns"):
await self.resolveDnsAddress(address)
await self.resolveOneAddress(address)
of multiCodec("dns4"):
await self.resolveDnsAddress(address, Domain.AF_INET)
await self.resolveOneAddress(address, Domain.AF_INET)
of multiCodec("dns6"):
await self.resolveDnsAddress(address, Domain.AF_INET6)
await self.resolveOneAddress(address, Domain.AF_INET6)
of multiCodec("dnsaddr"):
await self.resolveDnsAddr(address)
else:
doAssert false
@[address]
for ad in seq:
res.incl(ad)

9
libp2p/peerid.nim
View File

@ -43,7 +43,7 @@ func shortLog*(pid: PeerId): string =
var spid = $pid
if len(spid) > 10:
spid[3] = '*'
when (NimMajor, NimMinor) > (1, 4):
spid.delete(4 .. spid.high - 6)
else:
@ -148,7 +148,7 @@ func init*(pid: var PeerId, data: string): bool =
if Base58.decode(data, p, length) == Base58Status.Success:
p.setLen(length)
var opid: PeerId
shallowCopy(opid.data, p)
opid.data = p
if opid.validate():
pid = opid
result = true
@ -184,6 +184,11 @@ func init*(t: typedesc[PeerId], seckey: PrivateKey): Result[PeerId, cstring] =
## Create new peer id from private key ``seckey``.
PeerId.init(? seckey.getPublicKey().orError(cstring("invalid private key")))
proc random*(t: typedesc[PeerId], rng = newRng()): Result[PeerId, cstring] =
## Create new peer id with random public key.
let randomKey = PrivateKey.random(Secp256k1, rng[])[]
PeerId.init(randomKey).orError(cstring("failed to generate random key"))
func match*(pid: PeerId, pubkey: PublicKey): bool =
## Returns ``true`` if ``pid`` matches public key ``pubkey``.
let p = PeerId.init(pubkey)

58
libp2p/peerinfo.nim
View File

@ -15,18 +15,24 @@ else:
import std/[options, sequtils]
import pkg/[chronos, chronicles, stew/results]
import peerid, multiaddress, crypto/crypto, routing_record, errors, utility
import peerid, multiaddress, multicodec, crypto/crypto, routing_record, errors, utility
export peerid, multiaddress, crypto, routing_record, errors, results
## Our local peer info
type
PeerInfoError* = LPError
PeerInfoError* = object of LPError
AddressMapper* =
proc(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]]
{.gcsafe, raises: [Defect].}
PeerInfo* {.public.} = ref object
peerId*: PeerId
addrs*: seq[MultiAddress]
listenAddrs*: seq[MultiAddress]
addrs: seq[MultiAddress]
addressMappers*: seq[AddressMapper]
protocols*: seq[string]
protoVersion*: string
agentVersion*: string
@ -37,6 +43,7 @@ type
func shortLog*(p: PeerInfo): auto =
(
peerId: $p.peerId,
listenAddrs: mapIt(p.listenAddrs, $it),
addrs: mapIt(p.addrs, $it),
protocols: mapIt(p.protocols, $it),
protoVersion: p.protoVersion,
@ -44,7 +51,11 @@ func shortLog*(p: PeerInfo): auto =
)
chronicles.formatIt(PeerInfo): shortLog(it)
proc update*(p: PeerInfo) =
proc update*(p: PeerInfo) {.async.} =
p.addrs = p.listenAddrs
for mapper in p.addressMappers:
p.addrs = await mapper(p.addrs)
let sprRes = SignedPeerRecord.init(
p.privateKey,
PeerRecord.init(p.peerId, p.addrs)
@ -55,20 +66,46 @@ proc update*(p: PeerInfo) =
discard
#info "Can't update the signed peer record"
proc addrs*(p: PeerInfo): seq[MultiAddress] =
p.addrs
proc fullAddrs*(p: PeerInfo): MaResult[seq[MultiAddress]] =
let peerIdPart = ? MultiAddress.init(multiCodec("p2p"), p.peerId.data)
var res: seq[MultiAddress]
for address in p.addrs:
res.add(? concat(address, peerIdPart))
ok(res)
proc parseFullAddress*(ma: MultiAddress): MaResult[(PeerId, MultiAddress)] =
let p2pPart = ? ma[^1]
if ? p2pPart.protoCode != multiCodec("p2p"):
return err("Missing p2p part from multiaddress!")
let res = (
? PeerId.init(? p2pPart.protoArgument()).orErr("invalid peerid"),
? ma[0 .. ^2]
)
ok(res)
proc parseFullAddress*(ma: string | seq[byte]): MaResult[(PeerId, MultiAddress)] =
parseFullAddress(? MultiAddress.init(ma))
proc new*(
p: typedesc[PeerInfo],
key: PrivateKey,
addrs: openArray[MultiAddress] = [],
listenAddrs: openArray[MultiAddress] = [],
protocols: openArray[string] = [],
protoVersion: string = "",
agentVersion: string = ""): PeerInfo
{.raises: [Defect, PeerInfoError].} =
agentVersion: string = "",
addressMappers = newSeq[AddressMapper](),
): PeerInfo
{.raises: [Defect, LPError].} =
let pubkey = try:
key.getPublicKey().tryGet()
except CatchableError:
raise newException(PeerInfoError, "invalid private key")
let peerId = PeerId.init(key).tryGet()
let peerInfo = PeerInfo(
@ -77,10 +114,9 @@ proc new*(
privateKey: key,
protoVersion: protoVersion,
agentVersion: agentVersion,
addrs: @addrs,
listenAddrs: @listenAddrs,
protocols: @protocols,
addressMappers: addressMappers
)
peerInfo.update()
return peerInfo

6
libp2p/peerstore.nim
View File

@ -16,8 +16,7 @@ runnableExamples:
# Create a custom book type
type MoodBook = ref object of PeerBook[string]
var somePeerId: PeerId
discard somePeerId.init("")
var somePeerId = PeerId.random().get()
peerStore[MoodBook][somePeerId] = "Happy"
doAssert peerStore[MoodBook][somePeerId] == "Happy"
@ -153,6 +152,9 @@ proc updatePeerInfo*(
if info.addrs.len > 0:
peerStore[AddressBook][info.peerId] = info.addrs
if info.pubkey.isSome:
peerStore[KeyBook][info.peerId] = info.pubkey.get()
if info.agentVersion.isSome:
peerStore[AgentBook][info.peerId] = info.agentVersion.get().string

23
libp2p/protobuf/minprotobuf.nim
View File

@ -19,8 +19,7 @@ export results, utility
{.push public.}
const
MaxMessageSize* = 1'u shl 22
const MaxMessageSize = 1'u shl 22
type
ProtoFieldKind* = enum
@ -37,6 +36,7 @@ type
buffer*: seq[byte]
offset*: int
length*: int
maxSize*: uint
ProtoHeader* = object
wire*: ProtoFieldKind
@ -122,23 +122,28 @@ proc vsizeof*(field: ProtoField): int {.inline.} =
0
proc initProtoBuffer*(data: seq[byte], offset = 0,
options: set[ProtoFlags] = {}): ProtoBuffer =
options: set[ProtoFlags] = {},
maxSize = MaxMessageSize): ProtoBuffer =
## Initialize ProtoBuffer with shallow copy of ``data``.
shallowCopy(result.buffer, data)
result.buffer = data
result.offset = offset
result.options = options
result.maxSize = maxSize
proc initProtoBuffer*(data: openArray[byte], offset = 0,
options: set[ProtoFlags] = {}): ProtoBuffer =
options: set[ProtoFlags] = {},
maxSize = MaxMessageSize): ProtoBuffer =
## Initialize ProtoBuffer with copy of ``data``.
result.buffer = @data
result.offset = offset
result.options = options
result.maxSize = maxSize
proc initProtoBuffer*(options: set[ProtoFlags] = {}): ProtoBuffer =
proc initProtoBuffer*(options: set[ProtoFlags] = {}, maxSize = MaxMessageSize): ProtoBuffer =
## Initialize ProtoBuffer with new sequence of capacity ``cap``.
result.buffer = newSeq[byte]()
result.options = options
result.maxSize = maxSize
if WithVarintLength in options:
# Our buffer will start from position 10, so we can store length of buffer
# in [0, 9].
@ -335,7 +340,7 @@ proc skipValue(data: var ProtoBuffer, header: ProtoHeader): ProtoResult[void] =
var bsize = 0'u64
if PB.getUVarint(data.toOpenArray(), length, bsize).isOk():
data.offset += length
if bsize <= uint64(MaxMessageSize):
if bsize <= uint64(data.maxSize):
if data.isEnough(int(bsize)):
data.offset += int(bsize)
ok()
@ -399,7 +404,7 @@ proc getValue[T:byte|char](data: var ProtoBuffer, header: ProtoHeader,
outLength = 0
if PB.getUVarint(data.toOpenArray(), length, bsize).isOk():
data.offset += length
if bsize <= uint64(MaxMessageSize):
if bsize <= uint64(data.maxSize):
if data.isEnough(int(bsize)):
outLength = int(bsize)
if len(outBytes) >= int(bsize):
@ -427,7 +432,7 @@ proc getValue[T:seq[byte]|string](data: var ProtoBuffer, header: ProtoHeader,
if PB.getUVarint(data.toOpenArray(), length, bsize).isOk():
data.offset += length
if bsize <= uint64(MaxMessageSize):
if bsize <= uint64(data.maxSize):
if data.isEnough(int(bsize)):
outBytes.setLen(bsize)
if bsize > 0'u64:

324
libp2p/protocols/connectivity/autonat.nim Normal file
View File

@ -0,0 +1,324 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import std/[options, sets, sequtils]
import stew/results
import chronos, chronicles, stew/objects
import ../protocol,
../../switch,
../../multiaddress,
../../multicodec,
../../peerid,
../../utils/semaphore,
../../errors
logScope:
topics = "libp2p autonat"
const
AutonatCodec* = "/libp2p/autonat/1.0.0"
AddressLimit = 8
type
AutonatError* = object of LPError
AutonatUnreachableError* = object of LPError
MsgType* = enum
Dial = 0
DialResponse = 1
ResponseStatus* = enum
Ok = 0
DialError = 100
DialRefused = 101
BadRequest = 200
InternalError = 300
AutonatPeerInfo* = object
id*: Option[PeerId]
addrs*: seq[MultiAddress]
AutonatDial* = object
peerInfo*: Option[AutonatPeerInfo]
AutonatDialResponse* = object
status*: ResponseStatus
text*: Option[string]
ma*: Option[MultiAddress]
AutonatMsg* = object
msgType*: MsgType
dial*: Option[AutonatDial]
response*: Option[AutonatDialResponse]
proc encode*(msg: AutonatMsg): ProtoBuffer =
result = initProtoBuffer()
result.write(1, msg.msgType.uint)
if msg.dial.isSome():
var dial = initProtoBuffer()
if msg.dial.get().peerInfo.isSome():
var bufferPeerInfo = initProtoBuffer()
let peerInfo = msg.dial.get().peerInfo.get()
if peerInfo.id.isSome():
bufferPeerInfo.write(1, peerInfo.id.get())
for ma in peerInfo.addrs:
bufferPeerInfo.write(2, ma.data.buffer)
bufferPeerInfo.finish()
dial.write(1, bufferPeerInfo.buffer)
dial.finish()
result.write(2, dial.buffer)
if msg.response.isSome():
var bufferResponse = initProtoBuffer()
let response = msg.response.get()
bufferResponse.write(1, response.status.uint)
if response.text.isSome():
bufferResponse.write(2, response.text.get())
if response.ma.isSome():
bufferResponse.write(3, response.ma.get())
bufferResponse.finish()
result.write(3, bufferResponse.buffer)
result.finish()
proc encode*(d: AutonatDial): ProtoBuffer =
result = initProtoBuffer()
result.write(1, MsgType.Dial.uint)
var dial = initProtoBuffer()
if d.peerInfo.isSome():
var bufferPeerInfo = initProtoBuffer()
let peerInfo = d.peerInfo.get()
if peerInfo.id.isSome():
bufferPeerInfo.write(1, peerInfo.id.get())
for ma in peerInfo.addrs:
bufferPeerInfo.write(2, ma.data.buffer)
bufferPeerInfo.finish()
dial.write(1, bufferPeerInfo.buffer)
dial.finish()
result.write(2, dial.buffer)
result.finish()
proc encode*(r: AutonatDialResponse): ProtoBuffer =
result = initProtoBuffer()
result.write(1, MsgType.DialResponse.uint)
var bufferResponse = initProtoBuffer()
bufferResponse.write(1, r.status.uint)
if r.text.isSome():
bufferResponse.write(2, r.text.get())
if r.ma.isSome():
bufferResponse.write(3, r.ma.get())
bufferResponse.finish()
result.write(3, bufferResponse.buffer)
result.finish()
proc decode*(_: typedesc[AutonatMsg], buf: seq[byte]): Option[AutonatMsg] =
var
msgTypeOrd: uint32
pbDial: ProtoBuffer
pbResponse: ProtoBuffer
msg: AutonatMsg
let
pb = initProtoBuffer(buf)
r1 = pb.getField(1, msgTypeOrd)
r2 = pb.getField(2, pbDial)
r3 = pb.getField(3, pbResponse)
if r1.isErr() or r2.isErr() or r3.isErr(): return none(AutonatMsg)
if r1.get() and not checkedEnumAssign(msg.msgType, msgTypeOrd):
return none(AutonatMsg)
if r2.get():
var
pbPeerInfo: ProtoBuffer
dial: AutonatDial
let
r4 = pbDial.getField(1, pbPeerInfo)
if r4.isErr(): return none(AutonatMsg)
var peerInfo: AutonatPeerInfo
if r4.get():
var pid: PeerId
let
r5 = pbPeerInfo.getField(1, pid)
r6 = pbPeerInfo.getRepeatedField(2, peerInfo.addrs)
if r5.isErr() or r6.isErr(): return none(AutonatMsg)
if r5.get(): peerInfo.id = some(pid)
dial.peerInfo = some(peerInfo)
msg.dial = some(dial)
if r3.get():
var
statusOrd: uint
text: string
ma: MultiAddress
response: AutonatDialResponse
let
r4 = pbResponse.getField(1, statusOrd)
r5 = pbResponse.getField(2, text)
r6 = pbResponse.getField(3, ma)
if r4.isErr() or r5.isErr() or r6.isErr() or
(r4.get() and not checkedEnumAssign(response.status, statusOrd)):
return none(AutonatMsg)
if r5.get(): response.text = some(text)
if r6.get(): response.ma = some(ma)
msg.response = some(response)
return some(msg)
proc sendDial(conn: Connection, pid: PeerId, addrs: seq[MultiAddress]) {.async.} =
let pb = AutonatDial(peerInfo: some(AutonatPeerInfo(
id: some(pid),
addrs: addrs
))).encode()
await conn.writeLp(pb.buffer)
proc sendResponseError(conn: Connection, status: ResponseStatus, text: string = "") {.async.} =
let pb = AutonatDialResponse(
status: status,
text: if text == "": none(string) else: some(text),
ma: none(MultiAddress)
).encode()
await conn.writeLp(pb.buffer)
proc sendResponseOk(conn: Connection, ma: MultiAddress) {.async.} =
let pb = AutonatDialResponse(
status: ResponseStatus.Ok,
text: some("Ok"),
ma: some(ma)
).encode()
await conn.writeLp(pb.buffer)
type
Autonat* = ref object of LPProtocol
sem: AsyncSemaphore
switch*: Switch
dialTimeout: Duration
method dialMe*(a: Autonat, pid: PeerId, addrs: seq[MultiAddress] = newSeq[MultiAddress]()):
Future[MultiAddress] {.base, async.} =
proc getResponseOrRaise(autonatMsg: Option[AutonatMsg]): AutonatDialResponse {.raises: [UnpackError, AutonatError].} =
if autonatMsg.isNone() or
autonatMsg.get().msgType != DialResponse or
autonatMsg.get().response.isNone() or
(autonatMsg.get().response.get().status == Ok and
autonatMsg.get().response.get().ma.isNone()):
raise newException(AutonatError, "Unexpected response")
else:
autonatMsg.get().response.get()
let conn =
try:
if addrs.len == 0:
await a.switch.dial(pid, @[AutonatCodec])
else:
await a.switch.dial(pid, addrs, AutonatCodec)
except CatchableError as err:
raise newException(AutonatError, "Unexpected error when dialling", err)
defer: await conn.close()
await conn.sendDial(a.switch.peerInfo.peerId, a.switch.peerInfo.addrs)
let response = getResponseOrRaise(AutonatMsg.decode(await conn.readLp(1024)))
return case response.status:
of ResponseStatus.Ok:
response.ma.get()
of ResponseStatus.DialError:
raise newException(AutonatUnreachableError, "Peer could not dial us back")
else:
raise newException(AutonatError, "Bad status " & $response.status & " " & response.text.get(""))
proc tryDial(a: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} =
try:
await a.sem.acquire()
let ma = await a.switch.dialer.tryDial(conn.peerId, addrs).wait(a.dialTimeout)
if ma.isSome:
await conn.sendResponseOk(ma.get())
else:
await conn.sendResponseError(DialError, "Missing observed address")
except CancelledError as exc:
raise exc
except CatchableError as exc:
await conn.sendResponseError(DialError, exc.msg)
finally:
a.sem.release()
proc handleDial(a: Autonat, conn: Connection, msg: AutonatMsg): Future[void] =
if msg.dial.isNone() or msg.dial.get().peerInfo.isNone():
return conn.sendResponseError(BadRequest, "Missing Peer Info")
let peerInfo = msg.dial.get().peerInfo.get()
if peerInfo.id.isSome() and peerInfo.id.get() != conn.peerId:
return conn.sendResponseError(BadRequest, "PeerId mismatch")
if conn.observedAddr.isNone:
return conn.sendResponseError(BadRequest, "Missing observed address")
let observedAddr = conn.observedAddr.get()
var isRelayed = observedAddr.contains(multiCodec("p2p-circuit"))
if isRelayed.isErr() or isRelayed.get():
return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address")
let hostIp = observedAddr[0]
if hostIp.isErr() or not IP.match(hostIp.get()):
trace "wrong observed address", address=observedAddr
return conn.sendResponseError(InternalError, "Expected an IP address")
var addrs = initHashSet[MultiAddress]()
addrs.incl(observedAddr)
for ma in peerInfo.addrs:
isRelayed = ma.contains(multiCodec("p2p-circuit"))
if isRelayed.isErr() or isRelayed.get():
continue
let maFirst = ma[0]
if maFirst.isErr() or not IP.match(maFirst.get()):
continue
try:
addrs.incl(
if maFirst.get() == hostIp.get():
ma
else:
let maEnd = ma[1..^1]
if maEnd.isErr(): continue
hostIp.get() & maEnd.get()
)
except LPError as exc:
continue
if len(addrs) >= AddressLimit:
break
if len(addrs) == 0:
return conn.sendResponseError(DialRefused, "No dialable address")
return a.tryDial(conn, toSeq(addrs))
proc new*(T: typedesc[Autonat], switch: Switch, semSize: int = 1, dialTimeout = 15.seconds): T =
let autonat = T(switch: switch, sem: newAsyncSemaphore(semSize), dialTimeout: dialTimeout)
autonat.init()
autonat
method init*(a: Autonat) =
proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
try:
let msgOpt = AutonatMsg.decode(await conn.readLp(1024))
if msgOpt.isNone() or msgOpt.get().msgType != MsgType.Dial:
raise newException(AutonatError, "Received malformed message")
let msg = msgOpt.get()
await a.handleDial(conn, msg)
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "exception in autonat handler", exc = exc.msg, conn
finally:
trace "exiting autonat handler", conn
await conn.close()
a.handler = handleStream
a.codec = AutonatCodec

9
libp2p/protocols/relay/client.nim → libp2p/protocols/connectivity/relay/client.nim
View File

@ -20,11 +20,12 @@ import ./relay,
./messages,
./rconn,
./utils,
../../peerinfo,
../../switch,
../../multiaddress,
../../stream/connection
../../../peerinfo,
../../../switch,
../../../multiaddress,
../../../stream/connection
export options
logScope:
topics = "libp2p relay relay-client"

4
libp2p/protocols/relay/messages.nim → libp2p/protocols/connectivity/relay/messages.nim
View File

@ -14,8 +14,8 @@ else:
import options, macros, sequtils
import stew/objects
import ../../peerinfo,
../../signed_envelope
import ../../../peerinfo,
../../../signed_envelope
# Circuit Relay V1 Message

2
libp2p/protocols/relay/rconn.nim → libp2p/protocols/connectivity/relay/rconn.nim
View File

@ -14,7 +14,7 @@ else:
import chronos
import ../../stream/connection
import ../../../stream/connection
type
RelayConnection* = ref object of Connection

20
libp2p/protocols/relay/relay.nim → libp2p/protocols/connectivity/relay/relay.nim
View File

@ -19,16 +19,16 @@ import chronos, chronicles
import ./messages,
./rconn,
./utils,
../../peerinfo,
../../switch,
../../multiaddress,
../../multicodec,
../../stream/connection,
../../protocols/protocol,
../../transports/transport,
../../errors,
../../utils/heartbeat,
../../signed_envelope
../../../peerinfo,
../../../switch,
../../../multiaddress,
../../../multicodec,
../../../stream/connection,
../../../protocols/protocol,
../../../transports/transport,
../../../errors,
../../../utils/heartbeat,
../../../signed_envelope
# TODO:
# * Eventually replace std/times by chronos/timer. Currently chronos/timer

6
libp2p/protocols/relay/rtransport.nim → libp2p/protocols/connectivity/relay/rtransport.nim
View File

@ -19,9 +19,9 @@ import chronos, chronicles
import ./client,
./rconn,
./utils,
../../switch,
../../stream/connection,
../../transports/transport
../../../switch,
../../../stream/connection,
../../../transports/transport
logScope:
topics = "libp2p relay relay-transport"

16
libp2p/protocols/relay/utils.nim → libp2p/protocols/connectivity/relay/utils.nim
View File

@ -17,7 +17,7 @@ import options
import chronos, chronicles
import ./messages,
../../stream/connection
../../../stream/connection
logScope:
topics = "libp2p relay relay-utils"
@ -64,15 +64,17 @@ proc bridge*(connSrc: Connection, connDst: Connection) {.async.} =
await futSrc or futDst
if futSrc.finished():
bufRead = await futSrc
bytesSendFromSrcToDst.inc(bufRead)
await connDst.write(@bufSrcToDst[0..<bufRead])
zeroMem(addr(bufSrcToDst), bufSrcToDst.high + 1)
if bufRead > 0:
bytesSendFromSrcToDst.inc(bufRead)
await connDst.write(@bufSrcToDst[0..<bufRead])
zeroMem(addr(bufSrcToDst), bufSrcToDst.high + 1)
futSrc = connSrc.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
if futDst.finished():
bufRead = await futDst
bytesSendFromDstToSrc += bufRead
await connSrc.write(bufDstToSrc[0..<bufRead])
zeroMem(addr(bufDstToSrc), bufDstToSrc.high + 1)
if bufRead > 0:
bytesSendFromDstToSrc += bufRead
await connSrc.write(bufDstToSrc[0..<bufRead])
zeroMem(addr(bufDstToSrc), bufDstToSrc.high + 1)
futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
except CancelledError as exc:
raise exc

6
libp2p/protocols/identify.nim
View File

@ -16,6 +16,7 @@ else:
{.push raises: [].}
import std/[sequtils, options, strutils, sugar]
import stew/results
import chronos, chronicles
import ../protobuf/minprotobuf,
../peerinfo,
@ -80,7 +81,7 @@ chronicles.expandIt(IdentifyInfo):
if iinfo.signedPeerRecord.isSome(): "Some"
else: "None"
proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): ProtoBuffer
proc encodeMsg(peerInfo: PeerInfo, observedAddr: Opt[MultiAddress], sendSpr: bool): ProtoBuffer
{.raises: [Defect].} =
result = initProtoBuffer()
@ -91,7 +92,8 @@ proc encodeMsg(peerInfo: PeerInfo, observedAddr: MultiAddress, sendSpr: bool): P
result.write(2, ma.data.buffer)
for proto in peerInfo.protocols:
result.write(3, proto)
result.write(4, observedAddr.data.buffer)
if observedAddr.isSome:
result.write(4, observedAddr.get().data.buffer)
let protoVersion = ProtoVersion
result.write(5, protoVersion)
let agentVersion = if peerInfo.agentVersion.len <= 0:

26
libp2p/protocols/protocol.nim
View File

@ -12,9 +12,14 @@ when (NimMajor, NimMinor) < (1, 4):
else:
{.push raises: [].}
import chronos
import chronos, stew/results
import ../stream/connection
export results
const
DefaultMaxIncomingStreams* = 10
type
LPProtoHandler* = proc (
conn: Connection,
@ -26,11 +31,17 @@ type
codecs*: seq[string]
handler*: LPProtoHandler ## this handler gets invoked by the protocol negotiator
started*: bool
maxIncomingStreams: Opt[int]
method init*(p: LPProtocol) {.base, gcsafe.} = discard
method start*(p: LPProtocol) {.async, base.} = p.started = true
method stop*(p: LPProtocol) {.async, base.} = p.started = false
proc maxIncomingStreams*(p: LPProtocol): int =
p.maxIncomingStreams.get(DefaultMaxIncomingStreams)
proc `maxIncomingStreams=`*(p: LPProtocol, val: int) =
p.maxIncomingStreams = Opt.some(val)
func codec*(p: LPProtocol): string =
assert(p.codecs.len > 0, "Codecs sequence was empty!")
@ -40,3 +51,16 @@ func `codec=`*(p: LPProtocol, codec: string) =
# always insert as first codec
# if we use this abstraction
p.codecs.insert(codec, 0)
proc new*(
T: type LPProtocol,
codecs: seq[string],
handler: LPProtoHandler, # default(Opt[int]) or Opt.none(int) don't work on 1.2
maxIncomingStreams: Opt[int] | int = Opt[int]()): T =
T(
codecs: codecs,
handler: handler,
maxIncomingStreams:
when maxIncomingStreams is int: Opt.some(maxIncomingStreams)
else: maxIncomingStreams
)

9
libp2p/protocols/pubsub/gossipsub.nim
View File

@ -360,7 +360,7 @@ method rpcHandler*(g: GossipSub,
template sub: untyped = rpcMsg.subscriptions[i]
g.handleSubscribe(peer, sub.topic, sub.subscribe)
# the above call applied limtis to subs number
# the above call applied limits to subs number
# in gossipsub we want to apply scoring as well
if rpcMsg.subscriptions.len > g.topicsHigh:
debug "received an rpc message with an oversized amount of subscriptions", peer,
@ -435,6 +435,13 @@ method rpcHandler*(g: GossipSub,
if rpcMsg.control.isSome():
g.handleControl(peer, rpcMsg.control.unsafeGet())
# Now, check subscription to update the meshes if required
for i in 0..<min(g.topicsHigh, rpcMsg.subscriptions.len):
let topic = rpcMsg.subscriptions[i].topic
if topic in g.topics and g.mesh.peers(topic) < g.parameters.dLow:
# rebalance but don't update metrics here, we do that only in the heartbeat
g.rebalanceMesh(topic, metrics = nil)
g.updateMetrics(rpcMsg)
method onTopicSubscription*(g: GossipSub, topic: string, subscribed: bool) =

3
libp2p/protocols/pubsub/gossipsub/behavior.nim
View File

@ -19,6 +19,9 @@ import ".."/[pubsubpeer, peertable, timedcache, mcache, floodsub, pubsub]
import "../rpc"/[messages]
import "../../.."/[peerid, multiaddress, utility, switch, routing_record, signed_envelope, utils/heartbeat]
logScope:
topics = "libp2p gossipsub"
declareGauge(libp2p_gossipsub_cache_window_size, "the number of messages in the cache")
declareGauge(libp2p_gossipsub_peers_per_topic_mesh, "gossipsub peers per topic in mesh", labels = ["topic"])
declareGauge(libp2p_gossipsub_peers_per_topic_fanout, "gossipsub peers per topic in fanout", labels = ["topic"])

3
libp2p/protocols/pubsub/gossipsub/scoring.nim
View File

@ -18,6 +18,9 @@ import "."/[types]
import ".."/[pubsubpeer]
import "../../.."/[peerid, multiaddress, utility, switch, utils/heartbeat]
logScope:
topics = "libp2p gossipsub"
declareGauge(libp2p_gossipsub_peers_scores, "the scores of the peers in gossipsub", labels = ["agent"])
declareCounter(libp2p_gossipsub_bad_score_disconnection, "the number of peers disconnected by gossipsub", labels = ["agent"])
declareGauge(libp2p_gossipsub_peers_score_firstMessageDeliveries, "Detailed gossipsub scoring metric", labels = ["agent"])

6
libp2p/protocols/pubsub/gossipsub/types.nim
View File

@ -16,7 +16,7 @@ import chronos
import std/[tables, sets]
import ".."/[floodsub, peertable, mcache, pubsubpeer]
import "../rpc"/[messages]
import "../../.."/[peerid, multiaddress]
import "../../.."/[peerid, multiaddress, utility]
const
GossipSubCodec* = "/meshsub/1.1.0"
@ -65,7 +65,7 @@ type
meshFailurePenalty*: float64
invalidMessageDeliveries*: float64
TopicParams* = object
TopicParams* {.public.} = object
topicWeight*: float64
# p1
@ -102,7 +102,7 @@ type
appScore*: float64 # application specific score
behaviourPenalty*: float64 # the eventual penalty score
GossipSubParams* = object
GossipSubParams* {.public.} = object
explicit*: bool
pruneBackoff*: Duration
unsubscribeBackoff*: Duration

24
libp2p/protocols/pubsub/pubsub.nim
View File

@ -130,7 +130,7 @@ type
knownTopics*: HashSet[string]
method unsubscribePeer*(p: PubSub, peerId: PeerId) {.base.} =
method unsubscribePeer*(p: PubSub, peerId: PeerId) {.base, gcsafe.} =
## handle peer disconnects
##
@ -267,11 +267,11 @@ proc updateMetrics*(p: PubSub, rpcMsg: RPCMsg) =
method rpcHandler*(p: PubSub,
peer: PubSubPeer,
rpcMsg: RPCMsg): Future[void] {.base.} =
rpcMsg: RPCMsg): Future[void] {.base, async.} =
## Handler that must be overridden by concrete implementation
raiseAssert "Unimplemented"
method onNewPeer(p: PubSub, peer: PubSubPeer) {.base.} = discard
method onNewPeer(p: PubSub, peer: PubSubPeer) {.base, gcsafe.} = discard
method onPubSubPeerEvent*(p: PubSub, peer: PubSubPeer, event: PubSubPeerEvent) {.base, gcsafe.} =
# Peer event is raised for the send connection in particular
@ -292,19 +292,11 @@ proc getOrCreatePeer*(
proc getConn(): Future[Connection] {.async.} =
return await p.switch.dial(peerId, protos)
proc dropConn(peer: PubSubPeer) =
proc dropConnAsync(peer: PubSubPeer) {.async.} =
try:
await p.switch.disconnect(peer.peerId)
except CatchableError as exc: # never cancelled
trace "Failed to close connection", peer, error = exc.name, msg = exc.msg
asyncSpawn dropConnAsync(peer)
proc onEvent(peer: PubSubPeer, event: PubSubPeerEvent) {.gcsafe.} =
p.onPubSubPeerEvent(peer, event)
# create new pubsub peer
let pubSubPeer = PubSubPeer.new(peerId, getConn, dropConn, onEvent, protos[0], p.maxMessageSize)
let pubSubPeer = PubSubPeer.new(peerId, getConn, onEvent, protos[0], p.maxMessageSize)
debug "created new pubsub peer", peerId
p.peers[peerId] = pubSubPeer
@ -385,7 +377,7 @@ method handleConn*(p: PubSub,
finally:
await conn.closeWithEOF()
method subscribePeer*(p: PubSub, peer: PeerId) {.base.} =
method subscribePeer*(p: PubSub, peer: PeerId) {.base, gcsafe.} =
## subscribe to remote peer to receive/send pubsub
## messages
##
@ -408,7 +400,7 @@ proc updateTopicMetrics(p: PubSub, topic: string) =
libp2p_pubsub_topic_handlers.set(others, labelValues = ["other"])
method onTopicSubscription*(p: PubSub, topic: string, subscribed: bool) {.base.} =
method onTopicSubscription*(p: PubSub, topic: string, subscribed: bool) {.base, gcsafe.} =
# Called when subscribe is called the first time for a topic or unsubscribe
# removes the last handler
@ -441,7 +433,7 @@ proc unsubscribe*(p: PubSub, topics: openArray[TopicPair]) {.public.} =
for t in topics:
p.unsubscribe(t.topic, t.handler)
proc unsubscribeAll*(p: PubSub, topic: string) {.public.} =
proc unsubscribeAll*(p: PubSub, topic: string) {.public, gcsafe.} =
## unsubscribe every `handler` from `topic`
if topic notin p.topics:
debug "unsubscribeAll called for an unknown topic", topic
@ -503,7 +495,7 @@ method initPubSub*(p: PubSub)
method addValidator*(p: PubSub,
topic: varargs[string],
hook: ValidatorHandler) {.base, public.} =
hook: ValidatorHandler) {.base, public, gcsafe.} =
## Add a validator to a `topic`. Each new message received in this
## will be sent to `hook`. `hook` can return either `Accept`,
## `Ignore` or `Reject` (which can descore the peer)

11
libp2p/protocols/pubsub/pubsubpeer.nim
View File

@ -12,7 +12,8 @@ when (NimMajor, NimMinor) < (1, 4):
else:
{.push raises: [].}
import std/[sequtils, strutils, tables, hashes]
import std/[sequtils, strutils, tables, hashes, options]
import stew/results
import chronos, chronicles, nimcrypto/sha2, metrics
import rpc/[messages, message, protobuf],
../../peerid,
@ -51,7 +52,6 @@ type
PubSubPeer* = ref object of RootObj
getConn*: GetConn # callback to establish a new send connection
dropConn*: DropConn # Function pointer to use to drop connections
onEvent*: OnEvent # Connectivity updates for peer
codec*: string # the protocol that this peer joined from
sendConn*: Connection # cached send connection
@ -175,7 +175,7 @@ proc connectOnce(p: PubSubPeer): Future[void] {.async.} =
trace "Get new send connection", p, newConn
p.sendConn = newConn
p.address = some(p.sendConn.observedAddr)
p.address = if p.sendConn.observedAddr.isSome: some(p.sendConn.observedAddr.get) else: none(MultiAddress)
if p.onEvent != nil:
p.onEvent(p, PubSubPeerEvent(kind: PubSubPeerEventKind.Connected))
@ -206,9 +206,6 @@ proc connectImpl(p: PubSubPeer) {.async.} =
await connectOnce(p)
except CatchableError as exc: # never cancelled
debug "Could not establish send connection", msg = exc.msg
finally:
# drop the connection, else we end up with ghost peers
if p.dropConn != nil: p.dropConn(p)
proc connect*(p: PubSubPeer) =
asyncSpawn connectImpl(p)
@ -286,14 +283,12 @@ proc new*(
T: typedesc[PubSubPeer],
peerId: PeerId,
getConn: GetConn,
dropConn: DropConn,
onEvent: OnEvent,
codec: string,
maxMessageSize: int): T =
T(
getConn: getConn,
dropConn: dropConn,
onEvent: onEvent,
codec: codec,
peerId: peerId,

9
libp2p/protocols/pubsub/rpc/protobuf.nim
View File

@ -22,7 +22,7 @@ import messages,
logScope:
topics = "pubsubprotobuf"
topics = "libp2p pubsubprotobuf"
when defined(libp2p_protobuf_metrics):
import metrics
@ -304,14 +304,15 @@ proc decodeMessages*(pb: ProtoBuffer): ProtoResult[seq[Message]] {.inline.} =
if ? pb.getRepeatedField(2, msgpbs):
trace "decodeMessages: read messages", count = len(msgpbs)
for item in msgpbs:
msgs.add(? decodeMessage(initProtoBuffer(item)))
# size is constrained at the network level
msgs.add(? decodeMessage(initProtoBuffer(item, maxSize = uint.high)))
else:
trace "decodeMessages: no messages found"
ok(msgs)
proc encodeRpcMsg*(msg: RPCMsg, anonymize: bool): seq[byte] =
trace "encodeRpcMsg: encoding message", msg = msg.shortLog()
var pb = initProtoBuffer()
var pb = initProtoBuffer(maxSize = uint.high)
for item in msg.subscriptions:
pb.write(1, item)
for item in msg.messages:
@ -324,7 +325,7 @@ proc encodeRpcMsg*(msg: RPCMsg, anonymize: bool): seq[byte] =
proc decodeRpcMsg*(msg: seq[byte]): ProtoResult[RPCMsg] {.inline.} =
trace "decodeRpcMsg: decoding message", msg = msg.shortLog()
var pb = initProtoBuffer(msg)
var pb = initProtoBuffer(msg, maxSize = uint.high)
var rpcMsg = ok(RPCMsg())
assign(rpcMsg.get().messages, ? pb.decodeMessages())
assign(rpcMsg.get().subscriptions, ? pb.decodeSubscriptions())

677
libp2p/protocols/rendezvous.nim Normal file
View File

@ -0,0 +1,677 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import tables, sequtils, sugar, sets, options
import chronos,
chronicles,
bearssl/rand,
stew/[byteutils, objects]
import ./protocol,
../switch,
../routing_record,
../utils/heartbeat,
../stream/connection,
../utils/offsettedseq,
../utils/semaphore
export chronicles
logScope:
topics = "libp2p discovery rendezvous"
const
RendezVousCodec* = "/rendezvous/1.0.0"
MinimumDuration* = 2.hours
MaximumDuration = 72.hours
MinimumTTL = MinimumDuration.seconds.uint64
MaximumTTL = MaximumDuration.seconds.uint64
RegistrationLimitPerPeer = 1000
DiscoverLimit = 1000'u64
SemaphoreDefaultSize = 5
type
MessageType {.pure.} = enum
Register = 0
RegisterResponse = 1
Unregister = 2
Discover = 3
DiscoverResponse = 4
ResponseStatus = enum
Ok = 0
InvalidNamespace = 100
InvalidSignedPeerRecord = 101
InvalidTTL = 102
InvalidCookie = 103
NotAuthorized = 200
InternalError = 300
Unavailable = 400
Cookie = object
offset : uint64
ns : string
Register = object
ns : string
signedPeerRecord: seq[byte]
ttl: Option[uint64] # in seconds
RegisterResponse = object
status: ResponseStatus
text: Option[string]
ttl: Option[uint64] # in seconds
Unregister = object
ns: string
Discover = object
ns: string
limit: Option[uint64]
cookie: Option[seq[byte]]
DiscoverResponse = object
registrations: seq[Register]
cookie: Option[seq[byte]]
status: ResponseStatus
text: Option[string]
Message = object
msgType: MessageType
register: Option[Register]
registerResponse: Option[RegisterResponse]
unregister: Option[Unregister]
discover: Option[Discover]
discoverResponse: Option[DiscoverResponse]
proc encode(c: Cookie): ProtoBuffer =
result = initProtoBuffer()
result.write(1, c.offset)
result.write(2, c.ns)
result.finish()
proc encode(r: Register): ProtoBuffer =
result = initProtoBuffer()
result.write(1, r.ns)
result.write(2, r.signedPeerRecord)
if r.ttl.isSome():
result.write(3, r.ttl.get())
result.finish()
proc encode(rr: RegisterResponse): ProtoBuffer =
result = initProtoBuffer()
result.write(1, rr.status.uint)
if rr.text.isSome():
result.write(2, rr.text.get())
if rr.ttl.isSome():
result.write(3, rr.ttl.get())
result.finish()
proc encode(u: Unregister): ProtoBuffer =
result = initProtoBuffer()
result.write(1, u.ns)
result.finish()
proc encode(d: Discover): ProtoBuffer =
result = initProtoBuffer()
result.write(1, d.ns)
if d.limit.isSome():
result.write(2, d.limit.get())
if d.cookie.isSome():
result.write(3, d.cookie.get())
result.finish()
proc encode(d: DiscoverResponse): ProtoBuffer =
result = initProtoBuffer()
for reg in d.registrations:
result.write(1, reg.encode())
if d.cookie.isSome():
result.write(2, d.cookie.get())
result.write(3, d.status.uint)
if d.text.isSome():
result.write(4, d.text.get())
result.finish()
proc encode(msg: Message): ProtoBuffer =
result = initProtoBuffer()
result.write(1, msg.msgType.uint)
if msg.register.isSome():
result.write(2, msg.register.get().encode())
if msg.registerResponse.isSome():
result.write(3, msg.registerResponse.get().encode())
if msg.unregister.isSome():
result.write(4, msg.unregister.get().encode())
if msg.discover.isSome():
result.write(5, msg.discover.get().encode())
if msg.discoverResponse.isSome():
result.write(6, msg.discoverResponse.get().encode())
result.finish()
proc decode(_: typedesc[Cookie], buf: seq[byte]): Option[Cookie] =
var c: Cookie
let
pb = initProtoBuffer(buf)
r1 = pb.getRequiredField(1, c.offset)
r2 = pb.getRequiredField(2, c.ns)
if r1.isErr() or r2.isErr(): return none(Cookie)
some(c)
proc decode(_: typedesc[Register], buf: seq[byte]): Option[Register] =
var
r: Register
ttl: uint64
let
pb = initProtoBuffer(buf)
r1 = pb.getRequiredField(1, r.ns)
r2 = pb.getRequiredField(2, r.signedPeerRecord)
r3 = pb.getField(3, ttl)
if r1.isErr() or r2.isErr() or r3.isErr(): return none(Register)
if r3.get(): r.ttl = some(ttl)
some(r)
proc decode(_: typedesc[RegisterResponse], buf: seq[byte]): Option[RegisterResponse] =
var
rr: RegisterResponse
statusOrd: uint
text: string
ttl: uint64
let
pb = initProtoBuffer(buf)
r1 = pb.getRequiredField(1, statusOrd)
r2 = pb.getField(2, text)
r3 = pb.getField(3, ttl)
if r1.isErr() or r2.isErr() or r3.isErr() or
not checkedEnumAssign(rr.status, statusOrd): return none(RegisterResponse)
if r2.get(): rr.text = some(text)
if r3.get(): rr.ttl = some(ttl)
some(rr)
proc decode(_: typedesc[Unregister], buf: seq[byte]): Option[Unregister] =
var u: Unregister
let
pb = initProtoBuffer(buf)
r1 = pb.getRequiredField(1, u.ns)
if r1.isErr(): return none(Unregister)
some(u)
proc decode(_: typedesc[Discover], buf: seq[byte]): Option[Discover] =
var
d: Discover
limit: uint64
cookie: seq[byte]
let
pb = initProtoBuffer(buf)
r1 = pb.getRequiredField(1, d.ns)
r2 = pb.getField(2, limit)
r3 = pb.getField(3, cookie)
if r1.isErr() or r2.isErr() or r3.isErr: return none(Discover)
if r2.get(): d.limit = some(limit)
if r3.get(): d.cookie = some(cookie)
some(d)
proc decode(_: typedesc[DiscoverResponse], buf: seq[byte]): Option[DiscoverResponse] =
var
dr: DiscoverResponse
registrations: seq[seq[byte]]
cookie: seq[byte]
statusOrd: uint
text: string
let
pb = initProtoBuffer(buf)
r1 = pb.getRepeatedField(1, registrations)
r2 = pb.getField(2, cookie)
r3 = pb.getRequiredField(3, statusOrd)
r4 = pb.getField(4, text)
if r1.isErr() or r2.isErr() or r3.isErr or r4.isErr() or
not checkedEnumAssign(dr.status, statusOrd): return none(DiscoverResponse)
for reg in registrations:
var r: Register
let regOpt = Register.decode(reg)
if regOpt.isNone(): return none(DiscoverResponse)
dr.registrations.add(regOpt.get())
if r2.get(): dr.cookie = some(cookie)
if r4.get(): dr.text = some(text)
some(dr)
proc decode(_: typedesc[Message], buf: seq[byte]): Option[Message] =
var
msg: Message
statusOrd: uint
pbr, pbrr, pbu, pbd, pbdr: ProtoBuffer
let
pb = initProtoBuffer(buf)
r1 = pb.getRequiredField(1, statusOrd)
r2 = pb.getField(2, pbr)
r3 = pb.getField(3, pbrr)
r4 = pb.getField(4, pbu)
r5 = pb.getField(5, pbd)
r6 = pb.getField(6, pbdr)
if r1.isErr() or r2.isErr() or r3.isErr() or
r4.isErr() or r5.isErr() or r6.isErr() or
not checkedEnumAssign(msg.msgType, statusOrd): return none(Message)
if r2.get():
msg.register = Register.decode(pbr.buffer)
if msg.register.isNone(): return none(Message)
if r3.get():
msg.registerResponse = RegisterResponse.decode(pbrr.buffer)
if msg.registerResponse.isNone(): return none(Message)
if r4.get():
msg.unregister = Unregister.decode(pbu.buffer)
if msg.unregister.isNone(): return none(Message)
if r5.get():
msg.discover = Discover.decode(pbd.buffer)
if msg.discover.isNone(): return none(Message)
if r6.get():
msg.discoverResponse = DiscoverResponse.decode(pbdr.buffer)
if msg.discoverResponse.isNone(): return none(Message)
some(msg)
type
RendezVousError* = object of LPError
RegisteredData = object
expiration: Moment
peerId: PeerId
data: Register
RendezVous* = ref object of LPProtocol
# Registered needs to be an offsetted sequence
# because we need stable index for the cookies.
registered: OffsettedSeq[RegisteredData]
# Namespaces is a table whose key is a salted namespace and
# the value is the index sequence corresponding to this
# namespace in the offsettedqueue.
namespaces: Table[string, seq[int]]
rng: ref HmacDrbgContext
salt: string
defaultDT: Moment
registerDeletionLoop: Future[void]
#registerEvent: AsyncEvent # TODO: to raise during the heartbeat
# + make the heartbeat sleep duration "smarter"
sema: AsyncSemaphore
peers: seq[PeerId]
cookiesSaved: Table[PeerId, Table[string, seq[byte]]]
switch: Switch
proc checkPeerRecord(spr: seq[byte], peerId: PeerId): Result[void, string] =
if spr.len == 0: return err("Empty peer record")
let signedEnv = ? SignedPeerRecord.decode(spr).mapErr(x => $x)
if signedEnv.data.peerId != peerId:
return err("Bad Peer ID")
return ok()
proc sendRegisterResponse(conn: Connection,
ttl: uint64) {.async.} =
let msg = encode(Message(
msgType: MessageType.RegisterResponse,
registerResponse: some(RegisterResponse(status: Ok, ttl: some(ttl)))))
await conn.writeLp(msg.buffer)
proc sendRegisterResponseError(conn: Connection,
status: ResponseStatus,
text: string = "") {.async.} =
let msg = encode(Message(
msgType: MessageType.RegisterResponse,
registerResponse: some(RegisterResponse(status: status, text: some(text)))))
await conn.writeLp(msg.buffer)
proc sendDiscoverResponse(conn: Connection,
s: seq[Register],
cookie: Cookie) {.async.} =
let msg = encode(Message(
msgType: MessageType.DiscoverResponse,
discoverResponse: some(DiscoverResponse(
status: Ok,
registrations: s,
cookie: some(cookie.encode().buffer)
))
))
await conn.writeLp(msg.buffer)
proc sendDiscoverResponseError(conn: Connection,
status: ResponseStatus,
text: string = "") {.async.} =
let msg = encode(Message(
msgType: MessageType.DiscoverResponse,
discoverResponse: some(DiscoverResponse(status: status, text: some(text)))))
await conn.writeLp(msg.buffer)
proc countRegister(rdv: RendezVous, peerId: PeerId): int =
let n = Moment.now()
for data in rdv.registered:
if data.peerId == peerId and data.expiration > n:
result.inc()
proc save(rdv: RendezVous,
ns: string,
peerId: PeerId,
r: Register,
update: bool = true) =
let nsSalted = ns & rdv.salt
discard rdv.namespaces.hasKeyOrPut(nsSalted, newSeq[int]())
try:
for index in rdv.namespaces[nsSalted]:
if rdv.registered[index].peerId == peerId:
if update == false: return
rdv.registered[index].expiration = rdv.defaultDT
rdv.registered.add(
RegisteredData(
peerId: peerId,
expiration: Moment.now() + r.ttl.get(MinimumTTL).int64.seconds,
data: r
)
)
rdv.namespaces[nsSalted].add(rdv.registered.high)
# rdv.registerEvent.fire()
except KeyError:
doAssert false, "Should have key"
proc register(rdv: RendezVous, conn: Connection, r: Register): Future[void] =
trace "Received Register", peerId = conn.peerId, ns = r.ns
if r.ns.len notin 1..255:
return conn.sendRegisterResponseError(InvalidNamespace)
let ttl = r.ttl.get(MinimumTTL)
if ttl notin MinimumTTL..MaximumTTL:
return conn.sendRegisterResponseError(InvalidTTL)
let pr = checkPeerRecord(r.signedPeerRecord, conn.peerId)
if pr.isErr():
return conn.sendRegisterResponseError(InvalidSignedPeerRecord, pr.error())
if rdv.countRegister(conn.peerId) >= RegistrationLimitPerPeer:
return conn.sendRegisterResponseError(NotAuthorized, "Registration limit reached")
rdv.save(r.ns, conn.peerId, r)
conn.sendRegisterResponse(ttl)
proc unregister(rdv: RendezVous, conn: Connection, u: Unregister) =
trace "Received Unregister", peerId = conn.peerId, ns = u.ns
let nsSalted = u.ns & rdv.salt
try:
for index in rdv.namespaces[nsSalted]:
if rdv.registered[index].peerId == conn.peerId:
rdv.registered[index].expiration = rdv.defaultDT
except KeyError:
return
proc discover(rdv: RendezVous, conn: Connection, d: Discover) {.async.} =
trace "Received Discover", peerId = conn.peerId, ns = d.ns
if d.ns.len notin 0..255:
await conn.sendDiscoverResponseError(InvalidNamespace)
return
var limit = min(DiscoverLimit, d.limit.get(DiscoverLimit))
var
cookie =
if d.cookie.isSome():
try:
Cookie.decode(d.cookie.get()).get()
except CatchableError:
await conn.sendDiscoverResponseError(InvalidCookie)
return
else: Cookie(offset: rdv.registered.low().uint64 - 1)
if cookie.ns != d.ns or
cookie.offset notin rdv.registered.low().uint64..rdv.registered.high().uint64:
cookie = Cookie(offset: rdv.registered.low().uint64 - 1)
let
nsSalted = d.ns & rdv.salt
namespaces =
if d.ns != "":
try:
rdv.namespaces[nsSalted]
except KeyError:
await conn.sendDiscoverResponseError(InvalidNamespace)
return
else: toSeq(cookie.offset.int..rdv.registered.high())
if namespaces.len() == 0:
await conn.sendDiscoverResponse(@[], Cookie())
return
var offset = namespaces[^1]
let n = Moment.now()
var s = collect(newSeq()):
for index in namespaces:
var reg = rdv.registered[index]
if limit == 0:
offset = index
break
if reg.expiration < n or index.uint64 <= cookie.offset: continue
limit.dec()
reg.data.ttl = some((reg.expiration - Moment.now()).seconds.uint64)
reg.data
rdv.rng.shuffle(s)
await conn.sendDiscoverResponse(s, Cookie(offset: offset.uint64, ns: d.ns))
proc advertisePeer(rdv: RendezVous,
peer: PeerId,
msg: seq[byte]) {.async.} =
proc advertiseWrap() {.async.} =
try:
let conn = await rdv.switch.dial(peer, RendezVousCodec)
defer: await conn.close()
await conn.writeLp(msg)
let
buf = await conn.readLp(4096)
msgRecv = Message.decode(buf).get()
if msgRecv.msgType != MessageType.RegisterResponse:
trace "Unexpected register response", peer, msgType = msgRecv.msgType
elif msgRecv.registerResponse.isNone() or
msgRecv.registerResponse.get().status != ResponseStatus.Ok:
trace "Refuse to register", peer, response = msgRecv.registerResponse
except CatchableError as exc:
trace "exception in the advertise", error = exc.msg
finally:
rdv.sema.release()
await rdv.sema.acquire()
discard await advertiseWrap().withTimeout(5.seconds)
proc advertise*(rdv: RendezVous,
ns: string,
ttl: Duration = MinimumDuration) {.async.} =
let sprBuff = rdv.switch.peerInfo.signedPeerRecord.encode()
if sprBuff.isErr():
raise newException(RendezVousError, "Wrong Signed Peer Record")
if ns.len notin 1..255:
raise newException(RendezVousError, "Invalid namespace")
if ttl notin MinimumDuration..MaximumDuration:
raise newException(RendezVousError, "Invalid time to live")
let
r = Register(ns: ns, signedPeerRecord: sprBuff.get(), ttl: some(ttl.seconds.uint64))
msg = encode(Message(msgType: MessageType.Register, register: some(r)))
rdv.save(ns, rdv.switch.peerInfo.peerId, r)
let fut = collect(newSeq()):
for peer in rdv.peers:
trace "Send Advertise", peerId = peer, ns
rdv.advertisePeer(peer, msg.buffer)
await allFutures(fut)
proc requestLocally*(rdv: RendezVous, ns: string): seq[PeerRecord] =
let
nsSalted = ns & rdv.salt
n = Moment.now()
try:
collect(newSeq()):
for index in rdv.namespaces[nsSalted]:
if rdv.registered[index].expiration > n:
SignedPeerRecord.decode(rdv.registered[index].data.signedPeerRecord).get().data
except KeyError as exc:
@[]
proc request*(rdv: RendezVous,
ns: string,
l: int = DiscoverLimit.int): Future[seq[PeerRecord]] {.async.} =
let nsSalted = ns & rdv.salt
var
s: Table[PeerId, (PeerRecord, Register)]
limit: uint64
d = Discover(ns: ns)
if l <= 0 or l > DiscoverLimit.int:
raise newException(RendezVousError, "Invalid limit")
if ns.len notin 0..255:
raise newException(RendezVousError, "Invalid namespace")
limit = l.uint64
proc requestPeer(peer: PeerId) {.async.} =
let conn = await rdv.switch.dial(peer, RendezVousCodec)
defer: await conn.close()
d.limit = some(limit)
d.cookie =
try:
some(rdv.cookiesSaved[peer][ns])
except KeyError as exc:
none(seq[byte])
await conn.writeLp(encode(Message(
msgType: MessageType.Discover,
discover: some(d))).buffer)
let
buf = await conn.readLp(65536)
msgRcv = Message.decode(buf).get()
if msgRcv.msgType != MessageType.DiscoverResponse or
msgRcv.discoverResponse.isNone():
debug "Unexpected discover response", msgType = msgRcv.msgType
return
let resp = msgRcv.discoverResponse.get()
if resp.status != ResponseStatus.Ok:
trace "Cannot discover", ns, status = resp.status, text = resp.text
return
if resp.cookie.isSome() and resp.cookie.get().len < 1000:
if rdv.cookiesSaved.hasKeyOrPut(peer, {ns: resp.cookie.get()}.toTable):
rdv.cookiesSaved[peer][ns] = resp.cookie.get()
for r in resp.registrations:
if limit == 0: return
if r.ttl.isNone() or r.ttl.get() > MaximumTTL: continue
let sprRes = SignedPeerRecord.decode(r.signedPeerRecord)
if sprRes.isErr(): continue
let pr = sprRes.get().data
if s.hasKey(pr.peerId):
let (prSaved, rSaved) = s[pr.peerId]
if (prSaved.seqNo == pr.seqNo and rSaved.ttl.get() < r.ttl.get()) or
prSaved.seqNo < pr.seqNo:
s[pr.peerId] = (pr, r)
else:
s[pr.peerId] = (pr, r)
limit.dec()
for (_, r) in s.values():
rdv.save(ns, peer, r, false)
# copy to avoid resizes during the loop
let peers = rdv.peers
for peer in peers:
if limit == 0: break
if RendezVousCodec notin rdv.switch.peerStore[ProtoBook][peer]: continue
try:
trace "Send Request", peerId = peer, ns
await peer.requestPeer()
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "exception catch in request", error = exc.msg
return toSeq(s.values()).mapIt(it[0])
proc unsubscribeLocally*(rdv: RendezVous, ns: string) =
let nsSalted = ns & rdv.salt
try:
for index in rdv.namespaces[nsSalted]:
if rdv.registered[index].peerId == rdv.switch.peerInfo.peerId:
rdv.registered[index].expiration = rdv.defaultDT
except KeyError:
return
proc unsubscribe*(rdv: RendezVous, ns: string) {.async.} =
# TODO: find a way to improve this, maybe something similar to the advertise
if ns.len notin 1..255:
raise newException(RendezVousError, "Invalid namespace")
rdv.unsubscribeLocally(ns)
let msg = encode(Message(
msgType: MessageType.Unregister,
unregister: some(Unregister(ns: ns))))
proc unsubscribePeer(rdv: RendezVous, peerId: PeerId) {.async.} =
try:
let conn = await rdv.switch.dial(peerId, RendezVousCodec)
defer: await conn.close()
await conn.writeLp(msg.buffer)
except CatchableError as exc:
trace "exception while unsubscribing", error = exc.msg
for peer in rdv.peers:
discard await rdv.unsubscribePeer(peer).withTimeout(5.seconds)
proc setup*(rdv: RendezVous, switch: Switch) =
rdv.switch = switch
proc handlePeer(peerId: PeerId, event: PeerEvent) {.async.} =
if event.kind == PeerEventKind.Joined:
rdv.peers.add(peerId)
elif event.kind == PeerEventKind.Left:
rdv.peers.keepItIf(it != peerId)
rdv.switch.addPeerEventHandler(handlePeer, Joined)
rdv.switch.addPeerEventHandler(handlePeer, Left)
proc new*(T: typedesc[RendezVous],
rng: ref HmacDrbgContext = newRng()): T =
let rdv = T(
rng: rng,
salt: string.fromBytes(generateBytes(rng[], 8)),
registered: initOffsettedSeq[RegisteredData](1),
defaultDT: Moment.now() - 1.days,
#registerEvent: newAsyncEvent(),
sema: newAsyncSemaphore(SemaphoreDefaultSize)
)
logScope: topics = "libp2p discovery rendezvous"
proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
try:
let
buf = await conn.readLp(4096)
msg = Message.decode(buf).get()
case msg.msgType:
of MessageType.Register: await rdv.register(conn, msg.register.get())
of MessageType.RegisterResponse:
trace "Got an unexpected Register Response", response = msg.registerResponse
of MessageType.Unregister: rdv.unregister(conn, msg.unregister.get())
of MessageType.Discover: await rdv.discover(conn, msg.discover.get())
of MessageType.DiscoverResponse:
trace "Got an unexpected Discover Response", response = msg.discoverResponse
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "exception in rendezvous handler", error = exc.msg
finally:
await conn.close()
rdv.handler = handleStream
rdv.codec = RendezVousCodec
return rdv
proc new*(T: typedesc[RendezVous],
switch: Switch,
rng: ref HmacDrbgContext = newRng()): T =
let rdv = T.new(rng)
rdv.setup(switch)
return rdv
proc deletesRegister(rdv: RendezVous) {.async.} =
heartbeat "Register timeout", 1.minutes:
let n = Moment.now()
rdv.registered.flushIfIt(it.expiration < n)
for data in rdv.namespaces.mvalues():
data.keepItIf(it >= rdv.registered.offset)
method start*(rdv: RendezVous) {.async.} =
if not rdv.registerDeletionLoop.isNil:
warn "Starting rendezvous twice"
return
rdv.registerDeletionLoop = rdv.deletesRegister()
rdv.started = true
method stop*(rdv: RendezVous) {.async.} =
if rdv.registerDeletionLoop.isNil:
warn "Stopping rendezvous without starting it"
return
rdv.started = false
rdv.registerDeletionLoop.cancel()
rdv.registerDeletionLoop = nil

45
libp2p/protocols/secure/noise.nim
View File

@ -38,7 +38,7 @@ const
# https://godoc.org/github.com/libp2p/go-libp2p-noise#pkg-constants
NoiseCodec* = "/noise"
PayloadString = "noise-libp2p-static-key:"
PayloadString = toBytes("noise-libp2p-static-key:")
ProtocolXXName = "Noise_XX_25519_ChaChaPoly_SHA256"
@ -339,7 +339,6 @@ proc handshakeXXOutbound(
hs = HandshakeState.init()
try:
hs.ss.mixHash(p.commonPrologue)
hs.s = p.noiseKeys
@ -445,7 +444,6 @@ method readMessage*(sconn: NoiseConnection): Future[seq[byte]] {.async.} =
dumpMessage(sconn, FlowDirection.Incoming, [])
trace "Received 0-length message", sconn
proc encryptFrame(
sconn: NoiseConnection,
cipherFrame: var openArray[byte],
@ -506,7 +504,7 @@ method write*(sconn: NoiseConnection, message: seq[byte]): Future[void] =
# sequencing issues
sconn.stream.write(cipherFrames)
method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureConn] {.async.} =
method handshake*(p: Noise, conn: Connection, initiator: bool, peerId: Opt[PeerId]): Future[SecureConn] {.async.} =
trace "Starting Noise handshake", conn, initiator
let timeout = conn.timeout
@ -515,7 +513,7 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon
# https://github.com/libp2p/specs/tree/master/noise#libp2p-data-in-handshake-messages
let
signedPayload = p.localPrivateKey.sign(
PayloadString.toBytes & p.noiseKeys.publicKey.getBytes).tryGet()
PayloadString & p.noiseKeys.publicKey.getBytes).tryGet()
var
libp2pProof = initProtoBuffer()
@ -538,11 +536,9 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon
remoteSig: Signature
remoteSigBytes: seq[byte]
let r1 = remoteProof.getField(1, remotePubKeyBytes)
let r2 = remoteProof.getField(2, remoteSigBytes)
if r1.isErr() or not(r1.get()):
if not remoteProof.getField(1, remotePubKeyBytes).valueOr(false):
raise newException(NoiseHandshakeError, "Failed to deserialize remote public key bytes. (initiator: " & $initiator & ")")
if r2.isErr() or not(r2.get()):
if not remoteProof.getField(2, remoteSigBytes).valueOr(false):
raise newException(NoiseHandshakeError, "Failed to deserialize remote signature bytes. (initiator: " & $initiator & ")")
if not remotePubKey.init(remotePubKeyBytes):
@ -550,33 +546,34 @@ method handshake*(p: Noise, conn: Connection, initiator: bool): Future[SecureCon
if not remoteSig.init(remoteSigBytes):
raise newException(NoiseHandshakeError, "Failed to decode remote signature. (initiator: " & $initiator & ")")
let verifyPayload = PayloadString.toBytes & handshakeRes.rs.getBytes
let verifyPayload = PayloadString & handshakeRes.rs.getBytes
if not remoteSig.verify(verifyPayload, remotePubKey):
raise newException(NoiseHandshakeError, "Noise handshake signature verify failed.")
else:
trace "Remote signature verified", conn
if initiator:
let pid = PeerId.init(remotePubKey)
if not conn.peerId.validate():
raise newException(NoiseHandshakeError, "Failed to validate peerId.")
if pid.isErr or pid.get() != conn.peerId:
let pid = PeerId.init(remotePubKey).valueOr:
raise newException(NoiseHandshakeError, "Invalid remote peer id: " & $error)
trace "Remote peer id", pid = $pid
if peerId.isSome():
let targetPid = peerId.get()
if not targetPid.validate():
raise newException(NoiseHandshakeError, "Failed to validate expected peerId.")
if pid != targetPid:
var
failedKey: PublicKey
discard extractPublicKey(conn.peerId, failedKey)
debug "Noise handshake, peer infos don't match!",
discard extractPublicKey(targetPid, failedKey)
debug "Noise handshake, peer id doesn't match!",
initiator, dealt_peer = conn,
dealt_key = $failedKey, received_peer = $pid,
received_key = $remotePubKey
raise newException(NoiseHandshakeError, "Noise handshake, peer infos don't match! " & $pid & " != " & $conn.peerId)
else:
let pid = PeerId.init(remotePubKey)
if pid.isErr:
raise newException(NoiseHandshakeError, "Invalid remote peer id")
conn.peerId = pid.get()
raise newException(NoiseHandshakeError, "Noise handshake, peer id don't match! " & $pid & " != " & $targetPid)
conn.peerId = pid
var tmp = NoiseConnection.new(conn, conn.peerId, conn.observedAddr)
if initiator:
tmp.readCs = handshakeRes.cs2
tmp.writeCs = handshakeRes.cs1

13
libp2p/protocols/secure/secio.nim
View File

@ -291,7 +291,7 @@ proc transactMessage(conn: Connection,
await conn.write(msg)
return await conn.readRawMessage()
method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[SecureConn] {.async.} =
method handshake*(s: Secio, conn: Connection, initiator: bool, peerId: Opt[PeerId]): Future[SecureConn] {.async.} =
var
localNonce: array[SecioNonceSize, byte]
remoteNonce: seq[byte]
@ -342,9 +342,14 @@ method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[S
remotePeerId = PeerId.init(remotePubkey).tryGet()
# TODO: PeerId check against supplied PeerId
if not initiator:
conn.peerId = remotePeerId
if peerId.isSome():
let targetPid = peerId.get()
if not targetPid.validate():
raise newException(SecioError, "Failed to validate expected peerId.")
if remotePeerId != targetPid:
raise newException(SecioError, "Peer ids don't match!")
conn.peerId = remotePeerId
let order = getOrder(remoteBytesPubkey, localNonce, localBytesPubkey,
remoteNonce).tryGet()
trace "Remote proposal", schemes = remoteExchanges, ciphers = remoteCiphers,

21
libp2p/protocols/secure/secure.nim
View File

@ -7,12 +7,14 @@
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push gcsafe.}
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import std/[strformat]
import stew/results
import chronos, chronicles
import ../protocol,
../../stream/streamseq,
@ -21,7 +23,7 @@ import ../protocol,
../../peerinfo,
../../errors
export protocol
export protocol, results
logScope:
topics = "libp2p secure"
@ -48,7 +50,7 @@ chronicles.formatIt(SecureConn): shortLog(it)
proc new*(T: type SecureConn,
conn: Connection,
peerId: PeerId,
observedAddr: MultiAddress,
observedAddr: Opt[MultiAddress],
timeout: Duration = DefaultConnectionTimeout): T =
result = T(stream: conn,
peerId: peerId,
@ -79,13 +81,15 @@ method getWrapped*(s: SecureConn): Connection = s.stream
method handshake*(s: Secure,
conn: Connection,
initiator: bool): Future[SecureConn] {.async, base.} =
initiator: bool,
peerId: Opt[PeerId]): Future[SecureConn] {.async, base.} =
doAssert(false, "Not implemented!")
proc handleConn(s: Secure,
conn: Connection,
initiator: bool): Future[Connection] {.async.} =
var sconn = await s.handshake(conn, initiator)
initiator: bool,
peerId: Opt[PeerId]): Future[Connection] {.async.} =
var sconn = await s.handshake(conn, initiator, peerId)
# mark connection bottom level transport direction
# this is the safest place to do this
# we require this information in for example gossipsub
@ -121,7 +125,7 @@ method init*(s: Secure) =
try:
# We don't need the result but we
# definitely need to await the handshake
discard await s.handleConn(conn, false)
discard await s.handleConn(conn, false, Opt.none(PeerId))
trace "connection secured", conn
except CancelledError as exc:
warn "securing connection canceled", conn
@ -135,9 +139,10 @@ method init*(s: Secure) =
method secure*(s: Secure,
conn: Connection,
initiator: bool):
initiator: bool,
peerId: Opt[PeerId]):
Future[Connection] {.base.} =
s.handleConn(conn, initiator)
s.handleConn(conn, initiator, peerId)
method readOnce*(s: SecureConn,
pbytes: pointer,

159
libp2p/services/autonatservice.nim Normal file
View File

@ -0,0 +1,159 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import std/[options, deques, sequtils]
import chronos, metrics
import ../switch
import ../protocols/[connectivity/autonat]
import ../utils/heartbeat
import ../crypto/crypto
declarePublicGauge(libp2p_autonat_reachability_confidence, "autonat reachability confidence", labels = ["reachability"])
type
AutonatService* = ref object of Service
newConnectedPeerHandler: PeerEventHandler
scheduleHandle: Future[void]
networkReachability: NetworkReachability
confidence: Option[float]
answers: Deque[NetworkReachability]
autonat: Autonat
statusAndConfidenceHandler: StatusAndConfidenceHandler
rng: ref HmacDrbgContext
scheduleInterval: Option[Duration]
askNewConnectedPeers: bool
numPeersToAsk: int
maxQueueSize: int
minConfidence: float
dialTimeout: Duration
NetworkReachability* {.pure.} = enum
NotReachable, Reachable, Unknown
StatusAndConfidenceHandler* = proc (networkReachability: NetworkReachability, confidence: Option[float]): Future[void] {.gcsafe, raises: [Defect].}
proc new*(
T: typedesc[AutonatService],
autonat: Autonat,
rng: ref HmacDrbgContext,
scheduleInterval: Option[Duration] = none(Duration),
askNewConnectedPeers = true,
numPeersToAsk: int = 5,
maxQueueSize: int = 10,
minConfidence: float = 0.3,
dialTimeout = 30.seconds): T =
return T(
scheduleInterval: scheduleInterval,
networkReachability: Unknown,
confidence: none(float),
answers: initDeque[NetworkReachability](),
autonat: autonat,
rng: rng,
askNewConnectedPeers: askNewConnectedPeers,
numPeersToAsk: numPeersToAsk,
maxQueueSize: maxQueueSize,
minConfidence: minConfidence,
dialTimeout: dialTimeout)
proc networkReachability*(self: AutonatService): NetworkReachability {.inline.} =
return self.networkReachability
proc callHandler(self: AutonatService) {.async.} =
if not isNil(self.statusAndConfidenceHandler):
await self.statusAndConfidenceHandler(self.networkReachability, self.confidence)
proc handleAnswer(self: AutonatService, ans: NetworkReachability) {.async.} =
if ans == Unknown:
return
if self.answers.len == self.maxQueueSize:
self.answers.popFirst()
self.answers.addLast(ans)
self.networkReachability = Unknown
self.confidence = none(float)
const reachabilityPriority = [Reachable, NotReachable]
for reachability in reachabilityPriority:
let confidence = self.answers.countIt(it == reachability) / self.maxQueueSize
libp2p_autonat_reachability_confidence.set(value = confidence, labelValues = [$reachability])
if self.confidence.isNone and confidence >= self.minConfidence:
self.networkReachability = reachability
self.confidence = some(confidence)
trace "Current status", currentStats = $self.networkReachability, confidence = $self.confidence
proc askPeer(self: AutonatService, s: Switch, peerId: PeerId): Future[NetworkReachability] {.async.} =
trace "Asking for reachability", peerId = $peerId
let ans =
try:
discard await self.autonat.dialMe(peerId).wait(self.dialTimeout)
Reachable
except AutonatUnreachableError:
trace "dialMe answer is not reachable", peerId = $peerId
NotReachable
except AsyncTimeoutError:
trace "dialMe timed out", peerId = $peerId
Unknown
except CatchableError as err:
trace "dialMe unexpected error", peerId = $peerId, errMsg = $err.msg
Unknown
await self.handleAnswer(ans)
if not isNil(self.statusAndConfidenceHandler):
await self.statusAndConfidenceHandler(self.networkReachability, self.confidence)
return ans
proc askConnectedPeers(self: AutonatService, switch: Switch) {.async.} =
var peers = switch.connectedPeers(Direction.Out)
self.rng.shuffle(peers)
var answersFromPeers = 0
for peer in peers:
if answersFromPeers >= self.numPeersToAsk:
break
elif (await askPeer(self, switch, peer)) != Unknown:
answersFromPeers.inc()
proc schedule(service: AutonatService, switch: Switch, interval: Duration) {.async.} =
heartbeat "Schedule AutonatService run", interval:
await service.run(switch)
method setup*(self: AutonatService, switch: Switch): Future[bool] {.async.} =
let hasBeenSetup = await procCall Service(self).setup(switch)
if hasBeenSetup:
if self.askNewConnectedPeers:
self.newConnectedPeerHandler = proc (peerId: PeerId, event: PeerEvent): Future[void] {.async.} =
discard askPeer(self, switch, peerId)
await self.callHandler()
switch.connManager.addPeerEventHandler(self.newConnectedPeerHandler, PeerEventKind.Joined)
if self.scheduleInterval.isSome():
self.scheduleHandle = schedule(self, switch, self.scheduleInterval.get())
return hasBeenSetup
method run*(self: AutonatService, switch: Switch) {.async, public.} =
await askConnectedPeers(self, switch)
await self.callHandler()
method stop*(self: AutonatService, switch: Switch): Future[bool] {.async, public.} =
let hasBeenStopped = await procCall Service(self).stop(switch)
if hasBeenStopped:
if not isNil(self.scheduleHandle):
self.scheduleHandle.cancel()
self.scheduleHandle = nil
if not isNil(self.newConnectedPeerHandler):
switch.connManager.removePeerEventHandler(self.newConnectedPeerHandler, PeerEventKind.Joined)
return hasBeenStopped
proc statusAndConfidenceHandler*(self: AutonatService, statusAndConfidenceHandler: StatusAndConfidenceHandler) =
self.statusAndConfidenceHandler = statusAndConfidenceHandler

2
libp2p/stream/bufferstream.nim
View File

@ -79,7 +79,7 @@ method pushData*(s: BufferStream, data: seq[byte]) {.base, async.} =
&"Only one concurrent push allowed for stream {s.shortLog()}")
if s.isClosed or s.pushedEof:
raise newLPStreamEOFError()
raise newLPStreamClosedError()
if data.len == 0:
return # Don't push 0-length buffers, these signal EOF

8
libp2p/stream/chronosstream.nim
View File

@ -13,10 +13,13 @@ else:
{.push raises: [].}
import std/[oids, strformat]
import stew/results
import chronos, chronicles, metrics
import connection
import ../utility
export results
logScope:
topics = "libp2p chronosstream"
@ -60,7 +63,7 @@ proc init*(C: type ChronosStream,
client: StreamTransport,
dir: Direction,
timeout = DefaultChronosStreamTimeout,
observedAddr: MultiAddress = MultiAddress()): ChronosStream =
observedAddr: Opt[MultiAddress]): ChronosStream =
result = C(client: client,
timeout: timeout,
dir: dir,
@ -127,6 +130,9 @@ proc completeWrite(
method write*(s: ChronosStream, msg: seq[byte]): Future[void] =
# Avoid a copy of msg being kept in the closure created by `{.async.}` as this
# drives up memory usage
if msg.len == 0:
trace "Empty byte seq, nothing to write"
return
if s.closed:
let fut = newFuture[void]("chronosstream.write.closed")
fut.fail(newLPStreamClosedError())

9
libp2p/stream/connection.nim
View File

@ -13,13 +13,14 @@ else:
{.push raises: [].}
import std/[hashes, oids, strformat]
import stew/results
import chronicles, chronos, metrics
import lpstream,
../multiaddress,
../peerinfo,
../errors
export lpstream, peerinfo, errors
export lpstream, peerinfo, errors, results
logScope:
topics = "libp2p connection"
@ -37,7 +38,7 @@ type
timerTaskFut: Future[void] # the current timer instance
timeoutHandler*: TimeoutHandler # timeout handler
peerId*: PeerId
observedAddr*: MultiAddress
observedAddr*: Opt[MultiAddress]
upgraded*: Future[void]
protocol*: string # protocol used by the connection, used as tag for metrics
transportDir*: Direction # The bottom level transport (generally the socket) direction
@ -160,9 +161,9 @@ method getWrapped*(s: Connection): Connection {.base.} =
proc new*(C: type Connection,
peerId: PeerId,
dir: Direction,
observedAddr: Opt[MultiAddress],
timeout: Duration = DefaultConnectionTimeout,
timeoutHandler: TimeoutHandler = nil,
observedAddr: MultiAddress = MultiAddress()): Connection =
timeoutHandler: TimeoutHandler = nil): Connection =
result = C(peerId: peerId,
dir: dir,
timeout: timeout,

41
libp2p/stream/lpstream.nim
View File

@ -9,6 +9,7 @@
## Length Prefixed stream implementation
{.push gcsafe.}
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
@ -59,7 +60,18 @@ type
LPStreamWriteError* = object of LPStreamError
par*: ref CatchableError
LPStreamEOFError* = object of LPStreamError
LPStreamClosedError* = object of LPStreamError
# X | Read | Write
# Local close | Works | LPStreamClosedError
# Remote close | LPStreamRemoteClosedError | Works
# Local reset | LPStreamClosedError | LPStreamClosedError
# Remote reset | LPStreamResetError | LPStreamResetError
# Connection down | LPStreamConnDown | LPStreamConnDownError
LPStreamResetError* = object of LPStreamEOFError
LPStreamClosedError* = object of LPStreamEOFError
LPStreamRemoteClosedError* = object of LPStreamEOFError
LPStreamConnDownError* = object of LPStreamEOFError
InvalidVarintError* = object of LPStreamError
MaxSizeError* = object of LPStreamError
@ -68,7 +80,7 @@ type
opened*: uint64
closed*: uint64
proc setupStreamTracker(name: string): StreamTracker =
proc setupStreamTracker*(name: string): StreamTracker =
let tracker = new StreamTracker
proc dumpTracking(): string {.gcsafe.} =
@ -119,9 +131,22 @@ proc newLPStreamIncorrectDefect*(m: string): ref LPStreamIncorrectDefect =
proc newLPStreamEOFError*(): ref LPStreamEOFError =
result = newException(LPStreamEOFError, "Stream EOF!")
proc newLPStreamResetError*(): ref LPStreamResetError =
result = newException(LPStreamResetError, "Stream Reset!")
proc newLPStreamClosedError*(): ref LPStreamClosedError =
result = newException(LPStreamClosedError, "Stream Closed!")
proc newLPStreamRemoteClosedError*(): ref LPStreamRemoteClosedError =
result = newException(LPStreamRemoteClosedError, "Stream Remotely Closed!")
proc newLPStreamConnDownError*(
parentException: ref Exception = nil): ref LPStreamConnDownError =
result = newException(
LPStreamConnDownError,
"Stream Underlying Connection Closed!",
parentException)
func shortLog*(s: LPStream): auto =
if s.isNil: "LPStream(nil)"
else: $s.oid
@ -165,6 +190,8 @@ proc readExactly*(s: LPStream,
## Waits for `nbytes` to be available, then read
## them and return them
if s.atEof:
var ch: char
discard await s.readOnce(addr ch, 1)
raise newLPStreamEOFError()
if nbytes == 0:
@ -183,6 +210,10 @@ proc readExactly*(s: LPStream,
if read == 0:
doAssert s.atEof()
trace "couldn't read all bytes, stream EOF", s, nbytes, read
# Re-readOnce to raise a more specific error than EOF
# Raise EOF if it doesn't raise anything(shouldn't happen)
discard await s.readOnce(addr pbuffer[read], nbytes - read)
warn "Read twice while at EOF"
raise newLPStreamEOFError()
if read < nbytes:
@ -200,8 +231,7 @@ proc readLine*(s: LPStream,
while true:
var ch: char
if (await readOnce(s, addr ch, 1)) == 0:
raise newLPStreamEOFError()
await readExactly(s, addr ch, 1)
if sep[state] == ch:
inc(state)
@ -224,8 +254,7 @@ proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe, public.} =
buffer: array[10, byte]
for i in 0..<len(buffer):
if (await conn.readOnce(addr buffer[i], 1)) == 0:
raise newLPStreamEOFError()
await conn.readExactly(addr buffer[i], 1)
var
varint: uint64

72
libp2p/switch.nim
View File

@ -74,6 +74,28 @@ type
peerStore*: PeerStore
nameResolver*: NameResolver
started: bool
services*: seq[Service]
Service* = ref object of RootObj
inUse: bool
method setup*(self: Service, switch: Switch): Future[bool] {.base, async, gcsafe.} =
if self.inUse:
warn "service setup has already been called"
return false
self.inUse = true
return true
method run*(self: Service, switch: Switch) {.base, async, gcsafe.} =
doAssert(false, "Not implemented!")
method stop*(self: Service, switch: Switch): Future[bool] {.base, async, gcsafe.} =
if not self.inUse:
warn "service is already stopped"
return false
self.inUse = false
return true
proc addConnEventHandler*(s: Switch,
handler: ConnEventHandler,
@ -108,6 +130,9 @@ method addTransport*(s: Switch, t: Transport) =
s.transports &= t
s.dialer.addTransport(t)
proc connectedPeers*(s: Switch, dir: Direction): seq[PeerId] =
s.connManager.connectedPeers(dir)
proc isConnected*(s: Switch, peerId: PeerId): bool {.public.} =
## returns true if the peer has one or more
## associated connections
@ -128,6 +153,18 @@ method connect*(
s.dialer.connect(peerId, addrs, forceDial)
method connect*(
s: Switch,
address: MultiAddress,
allowUnknownPeerId = false): Future[PeerId] =
## Connects to a peer and retrieve its PeerId
##
## If the P2P part is missing from the MA and `allowUnknownPeerId` is set
## to true, this will discover the PeerId while connecting. This exposes
## you to MiTM attacks, so it shouldn't be used without care!
s.dialer.connect(address, allowUnknownPeerId)
method dial*(
s: Switch,
peerId: PeerId,
@ -282,6 +319,9 @@ proc stop*(s: Switch) {.async, public.} =
if not a.finished:
a.cancel()
for service in s.services:
discard await service.stop(s)
await s.ms.stop()
trace "Switch stopped"
@ -289,14 +329,18 @@ proc stop*(s: Switch) {.async, public.} =
proc start*(s: Switch) {.async, gcsafe, public.} =
## Start listening on every transport
if s.started:
warn "Switch has already been started"
return
trace "starting switch for peer", peerInfo = s.peerInfo
var startFuts: seq[Future[void]]
for t in s.transports:
let addrs = s.peerInfo.addrs.filterIt(
let addrs = s.peerInfo.listenAddrs.filterIt(
t.handles(it)
)
s.peerInfo.addrs.keepItIf(
s.peerInfo.listenAddrs.keepItIf(
it notin addrs
)
@ -305,22 +349,23 @@ proc start*(s: Switch) {.async, gcsafe, public.} =
await allFutures(startFuts)
for s in startFuts:
if s.failed:
# TODO: replace this exception with a `listenError` callback. See
# https://github.com/status-im/nim-libp2p/pull/662 for more info.
raise newException(transport.TransportError,
"Failed to start one transport", s.error)
for fut in startFuts:
if fut.failed:
await s.stop()
raise fut.error
for t in s.transports: # for each transport
if t.addrs.len > 0 or t.running:
s.acceptFuts.add(s.accept(t))
s.peerInfo.addrs &= t.addrs
s.peerInfo.listenAddrs &= t.addrs
s.peerInfo.update()
await s.peerInfo.update()
await s.ms.start()
for service in s.services:
discard await service.setup(s)
s.started = true
debug "Started libp2p node", peer = s.peerInfo
@ -332,7 +377,8 @@ proc newSwitch*(peerInfo: PeerInfo,
connManager: ConnManager,
ms: MultistreamSelect,
nameResolver: NameResolver = nil,
peerStore = PeerStore.new()): Switch
peerStore = PeerStore.new(),
services = newSeq[Service]()): Switch
{.raises: [Defect, LPError], public.} =
if secureManagers.len == 0:
raise newException(LPError, "Provide at least one secure manager")
@ -344,8 +390,10 @@ proc newSwitch*(peerInfo: PeerInfo,
connManager: connManager,
peerStore: peerStore,
dialer: Dialer.new(peerInfo.peerId, connManager, transports, ms, nameResolver),
nameResolver: nameResolver)
nameResolver: nameResolver,
services: services)
switch.connManager.peerStore = peerStore
switch.mount(identity)
return switch

21
libp2p/transports/tcptransport.nim
View File

@ -15,6 +15,7 @@ else:
{.push raises: [].}
import std/[oids, sequtils]
import stew/results
import chronos, chronicles
import transport,
../errors,
@ -31,7 +32,7 @@ import transport,
logScope:
topics = "libp2p tcptransport"
export transport
export transport, results
const
TcpTransportTrackerName* = "libp2p.tcptransport"
@ -71,18 +72,20 @@ proc setupTcpTransportTracker(): TcpTransportTracker =
result.isLeaked = leakTransport
addTracker(TcpTransportTrackerName, result)
proc connHandler*(self: TcpTransport,
client: StreamTransport,
dir: Direction): Future[Connection] {.async.} =
var observedAddr: MultiAddress = MultiAddress()
proc getObservedAddr(client: StreamTransport): Future[MultiAddress] {.async.} =
try:
observedAddr = MultiAddress.init(client.remoteAddress).tryGet()
return MultiAddress.init(client.remoteAddress).tryGet()
except CatchableError as exc:
trace "Failed to create observedAddr", exc = exc.msg
if not(isNil(client) and client.closed):
await client.closeWait()
raise exc
proc connHandler*(self: TcpTransport,
client: StreamTransport,
observedAddr: Opt[MultiAddress],
dir: Direction): Future[Connection] {.async.} =
trace "Handling tcp connection", address = $observedAddr,
dir = $dir,
clients = self.clients[Direction.In].len +
@ -222,7 +225,8 @@ method accept*(self: TcpTransport): Future[Connection] {.async, gcsafe.} =
self.acceptFuts[index] = self.servers[index].accept()
let transp = await finished
return await self.connHandler(transp, Direction.In)
let observedAddr = await getObservedAddr(transp)
return await self.connHandler(transp, Opt.some(observedAddr), Direction.In)
except TransportOsError as exc:
# TODO: it doesn't sound like all OS errors
# can be ignored, we should re-raise those
@ -250,7 +254,8 @@ method dial*(
let transp = await connect(address)
try:
return await self.connHandler(transp, Direction.Out)
let observedAddr = await getObservedAddr(transp)
return await self.connHandler(transp, Opt.some(observedAddr), Direction.Out)
except CatchableError as err:
await transp.closeWait()
raise err

281
libp2p/transports/tortransport.nim Normal file
View File

@ -0,0 +1,281 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## Tor transport implementation
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import std/strformat
import chronos, chronicles, strutils
import stew/[byteutils, endians2, results, objects]
import ../multicodec
import transport,
tcptransport,
../switch,
../builders,
../stream/[lpstream, connection, chronosstream],
../multiaddress,
../upgrademngrs/upgrade
const
IPTcp = mapAnd(IP, mapEq("tcp"))
IPv4Tcp = mapAnd(IP4, mapEq("tcp"))
IPv6Tcp = mapAnd(IP6, mapEq("tcp"))
DnsTcp = mapAnd(DNSANY, mapEq("tcp"))
Socks5ProtocolVersion = byte(5)
NMethods = byte(1)
type
TorTransport* = ref object of Transport
transportAddress: TransportAddress
tcpTransport: TcpTransport
Socks5AuthMethod* {.pure.} = enum
NoAuth = 0
GSSAPI = 1
UsernamePassword = 2
NoAcceptableMethod = 0xff
Socks5RequestCommand* {.pure.} = enum
Connect = 1, Bind = 2, UdpAssoc = 3
Socks5AddressType* {.pure.} = enum
IPv4 = 1, FQDN = 3, IPv6 = 4
Socks5ReplyType* {.pure.} = enum
Succeeded = (0, "Succeeded"), ServerFailure = (1, "Server Failure"),
ConnectionNotAllowed = (2, "Connection Not Allowed"), NetworkUnreachable = (3, "Network Unreachable"),
HostUnreachable = (4, "Host Unreachable"), ConnectionRefused = (5, "Connection Refused"),
TtlExpired = (6, "Ttl Expired"), CommandNotSupported = (7, "Command Not Supported"),
AddressTypeNotSupported = (8, "Address Type Not Supported")
TransportStartError* = object of transport.TransportError
Socks5Error* = object of CatchableError
Socks5AuthFailedError* = object of Socks5Error
Socks5VersionError* = object of Socks5Error
Socks5ServerReplyError* = object of Socks5Error
proc new*(
T: typedesc[TorTransport],
transportAddress: TransportAddress,
flags: set[ServerFlags] = {},
upgrade: Upgrade): T {.public.} =
## Creates a Tor transport
T(
transportAddress: transportAddress,
upgrader: upgrade,
tcpTransport: TcpTransport.new(flags, upgrade))
proc handlesDial(address: MultiAddress): bool {.gcsafe.} =
return Onion3.match(address) or TCP.match(address) or DNSANY.match(address)
proc handlesStart(address: MultiAddress): bool {.gcsafe.} =
return TcpOnion3.match(address)
proc connectToTorServer(
transportAddress: TransportAddress): Future[StreamTransport] {.async, gcsafe.} =
let transp = await connect(transportAddress)
try:
discard await transp.write(@[Socks5ProtocolVersion, NMethods, Socks5AuthMethod.NoAuth.byte])
let
serverReply = await transp.read(2)
socks5ProtocolVersion = serverReply[0]
serverSelectedMethod = serverReply[1]
if socks5ProtocolVersion != Socks5ProtocolVersion:
raise newException(Socks5VersionError, "Unsupported socks version")
if serverSelectedMethod != Socks5AuthMethod.NoAuth.byte:
raise newException(Socks5AuthFailedError, "Unsupported auth method")
return transp
except CatchableError as err:
await transp.closeWait()
raise err
proc readServerReply(transp: StreamTransport) {.async, gcsafe.} =
## The specification for this code is defined on
## [link text](https://www.rfc-editor.org/rfc/rfc1928#section-5)
## and [link text](https://www.rfc-editor.org/rfc/rfc1928#section-6).
let
portNumOctets = 2
ipV4NumOctets = 4
ipV6NumOctets = 16
firstFourOctets = await transp.read(4)
socks5ProtocolVersion = firstFourOctets[0]
serverReply = firstFourOctets[1]
if socks5ProtocolVersion != Socks5ProtocolVersion:
raise newException(Socks5VersionError, "Unsupported socks version")
if serverReply != Socks5ReplyType.Succeeded.byte:
var socks5ReplyType: Socks5ReplyType
if socks5ReplyType.checkedEnumAssign(serverReply):
raise newException(Socks5ServerReplyError, fmt"Server reply error: {socks5ReplyType}")
else:
raise newException(LPError, fmt"Unexpected server reply: {serverReply}")
let atyp = firstFourOctets[3]
case atyp:
of Socks5AddressType.IPv4.byte:
discard await transp.read(ipV4NumOctets + portNumOctets)
of Socks5AddressType.FQDN.byte:
let fqdnNumOctets = await transp.read(1)
discard await transp.read(int(uint8.fromBytes(fqdnNumOctets)) + portNumOctets)
of Socks5AddressType.IPv6.byte:
discard await transp.read(ipV6NumOctets + portNumOctets)
else:
raise newException(LPError, "Address not supported")
proc parseOnion3(address: MultiAddress): (byte, seq[byte], seq[byte]) {.raises: [Defect, LPError, ValueError].} =
var addressArray = ($address).split('/')
if addressArray.len < 2: raise newException(LPError, fmt"Onion address not supported {address}")
addressArray = addressArray[2].split(':')
if addressArray.len == 0: raise newException(LPError, fmt"Onion address not supported {address}")
let
addressStr = addressArray[0] & ".onion"
dstAddr = @(uint8(addressStr.len).toBytes()) & addressStr.toBytes()
dstPort = address.data.buffer[37..38]
return (Socks5AddressType.FQDN.byte, dstAddr, dstPort)
proc parseIpTcp(address: MultiAddress): (byte, seq[byte], seq[byte]) {.raises: [Defect, LPError, ValueError].} =
let (codec, atyp) =
if IPv4Tcp.match(address):
(multiCodec("ip4"), Socks5AddressType.IPv4.byte)
elif IPv6Tcp.match(address):
(multiCodec("ip6"), Socks5AddressType.IPv6.byte)
else:
raise newException(LPError, fmt"IP address not supported {address}")
let
dstAddr = address[codec].get().protoArgument().get()
dstPort = address[multiCodec("tcp")].get().protoArgument().get()
(atyp, dstAddr, dstPort)
proc parseDnsTcp(address: MultiAddress): (byte, seq[byte], seq[byte]) =
let
dnsAddress = address[multiCodec("dns")].get().protoArgument().get()
dstAddr = @(uint8(dnsAddress.len).toBytes()) & dnsAddress
dstPort = address[multiCodec("tcp")].get().protoArgument().get()
(Socks5AddressType.FQDN.byte, dstAddr, dstPort)
proc dialPeer(
transp: StreamTransport, address: MultiAddress) {.async, gcsafe.} =
let (atyp, dstAddr, dstPort) =
if Onion3.match(address):
parseOnion3(address)
elif IPTcp.match(address):
parseIpTcp(address)
elif DnsTcp.match(address):
parseDnsTcp(address)
else:
raise newException(LPError, fmt"Address not supported: {address}")
let reserved = byte(0)
let request = @[
Socks5ProtocolVersion,
Socks5RequestCommand.Connect.byte,
reserved,
atyp] & dstAddr & dstPort
discard await transp.write(request)
await readServerReply(transp)
method dial*(
self: TorTransport,
hostname: string,
address: MultiAddress): Future[Connection] {.async, gcsafe.} =
## dial a peer
##
if not handlesDial(address):
raise newException(LPError, fmt"Address not supported: {address}")
trace "Dialing remote peer", address = $address
let transp = await connectToTorServer(self.transportAddress)
try:
await dialPeer(transp, address)
return await self.tcpTransport.connHandler(transp, Opt.none(MultiAddress), Direction.Out)
except CatchableError as err:
await transp.closeWait()
raise err
method start*(
self: TorTransport,
addrs: seq[MultiAddress]) {.async.} =
## listen on the transport
##
var listenAddrs: seq[MultiAddress]
var onion3Addrs: seq[MultiAddress]
for i, ma in addrs:
if not handlesStart(ma):
warn "Invalid address detected, skipping!", address = ma
continue
let listenAddress = ma[0..1].get()
listenAddrs.add(listenAddress)
let onion3 = ma[multiCodec("onion3")].get()
onion3Addrs.add(onion3)
if len(listenAddrs) != 0 and len(onion3Addrs) != 0:
await procCall Transport(self).start(onion3Addrs)
await self.tcpTransport.start(listenAddrs)
else:
raise newException(TransportStartError, "Tor Transport couldn't start, no supported addr was provided.")
method accept*(self: TorTransport): Future[Connection] {.async, gcsafe.} =
## accept a new Tor connection
##
let conn = await self.tcpTransport.accept()
conn.observedAddr = Opt.none(MultiAddress)
return conn
method stop*(self: TorTransport) {.async, gcsafe.} =
## stop the transport
##
await procCall Transport(self).stop() # call base
await self.tcpTransport.stop()
method handles*(t: TorTransport, address: MultiAddress): bool {.gcsafe.} =
if procCall Transport(t).handles(address):
return handlesDial(address) or handlesStart(address)
type
TorSwitch* = ref object of Switch
proc new*(
T: typedesc[TorSwitch],
torServer: TransportAddress,
rng: ref HmacDrbgContext,
addresses: seq[MultiAddress] = @[],
flags: set[ServerFlags] = {}): TorSwitch
{.raises: [LPError, Defect], public.} =
var builder = SwitchBuilder.new()
.withRng(rng)
.withTransport(proc(upgr: Upgrade): Transport = TorTransport.new(torServer, flags, upgr))
if addresses.len != 0:
builder = builder.withAddresses(addresses)
let switch = builder.withMplex()
.withNoise()
.build()
let torSwitch = T(
peerInfo: switch.peerInfo,
ms: switch.ms,
transports: switch.transports,
connManager: switch.connManager,
peerStore: switch.peerStore,
dialer: Dialer.new(switch.peerInfo.peerId, switch.connManager, switch.transports, switch.ms, nil),
nameResolver: nil)
torSwitch.connManager.peerStore = switch.peerStore
return torSwitch
method addTransport*(s: TorSwitch, t: Transport) =
doAssert(false, "not implemented!")
method getTorTransport*(s: TorSwitch): Transport {.base.} =
return s.transports[0]

5
libp2p/transports/transport.nim
View File

@ -87,12 +87,13 @@ method upgradeIncoming*(
method upgradeOutgoing*(
self: Transport,
conn: Connection): Future[Connection] {.base, gcsafe.} =
conn: Connection,
peerId: Opt[PeerId]): Future[Connection] {.base, gcsafe.} =
## base upgrade method that the transport uses to perform
## transport specific upgrades
##
self.upgrader.upgradeOutgoing(conn)
self.upgrader.upgradeOutgoing(conn, peerId)
method handles*(
self: Transport,

24
libp2p/transports/wstransport.nim
View File

@ -15,6 +15,7 @@ else:
{.push raises: [].}
import std/[sequtils]
import stew/results
import chronos, chronicles
import transport,
../errors,
@ -31,22 +32,26 @@ import transport,
logScope:
topics = "libp2p wstransport"
export transport, websock
export transport, websock, results
const
WsTransportTrackerName* = "libp2p.wstransport"
DefaultHeadersTimeout = 3.seconds
type
WsStream = ref object of Connection
session: WSSession
method initStream*(s: WsStream) =
if s.objName.len == 0:
s.objName = "WsStream"
procCall Connection(s).initStream()
proc new*(T: type WsStream,
session: WSSession,
dir: Direction,
timeout = 10.minutes,
observedAddr: MultiAddress = MultiAddress()): T =
observedAddr: Opt[MultiAddress],
timeout = 10.minutes): T =
let stream = T(
session: session,
@ -128,7 +133,7 @@ method start*(
factories = self.factories,
rng = self.rng)
for i, ma in addrs:
let isWss =
if WSS.match(ma):
@ -221,8 +226,7 @@ proc connHandler(self: WsTransport,
await stream.close()
raise exc
let conn = WsStream.new(stream, dir)
conn.observedAddr = observedAddr
let conn = WsStream.new(stream, dir, Opt.some(observedAddr))
self.connections[dir].add(conn)
proc onClose() {.async.} =
@ -256,7 +260,7 @@ method accept*(self: WsTransport): Future[Connection] {.async, gcsafe.} =
try:
let
wstransp = await self.wsserver.handleRequest(req)
wstransp = await self.wsserver.handleRequest(req).wait(self.handshakeTimeout)
isSecure = self.httpservers[index].secure
return await self.connHandler(wstransp, isSecure, Direction.In)
@ -273,6 +277,8 @@ method accept*(self: WsTransport): Future[Connection] {.async, gcsafe.} =
debug "AsyncStream Error", exc = exc.msg
except TransportTooManyError as exc:
debug "Too many files opened", exc = exc.msg
except AsyncTimeoutError as exc:
debug "Timed out", exc = exc.msg
except TransportUseClosedError as exc:
debug "Server was closed", exc = exc.msg
raise newTransportClosedError(exc)

7
libp2p/upgrademngrs/muxedupgrade.nim
View File

@ -88,10 +88,11 @@ proc mux*(
method upgradeOutgoing*(
self: MuxedUpgrade,
conn: Connection): Future[Connection] {.async, gcsafe.} =
conn: Connection,
peerId: Opt[PeerId]): Future[Connection] {.async, gcsafe.} =
trace "Upgrading outgoing connection", conn
let sconn = await self.secure(conn) # secure the connection
let sconn = await self.secure(conn, peerId) # secure the connection
if isNil(sconn):
raise newException(UpgradeFailedError,
"unable to secure connection, stopping upgrade")
@ -129,7 +130,7 @@ method upgradeIncoming*(
var cconn = conn
try:
var sconn = await secure.secure(cconn, false)
var sconn = await secure.secure(cconn, false, Opt.none(PeerId))
if isNil(sconn):
return

9
libp2p/upgrademngrs/upgrade.nim
View File

@ -7,6 +7,7 @@
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push gcsafe.}
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
@ -47,12 +48,14 @@ method upgradeIncoming*(
method upgradeOutgoing*(
self: Upgrade,
conn: Connection): Future[Connection] {.base.} =
conn: Connection,
peerId: Opt[PeerId]): Future[Connection] {.base.} =
doAssert(false, "Not implemented!")
proc secure*(
self: Upgrade,
conn: Connection): Future[Connection] {.async, gcsafe.} =
conn: Connection,
peerId: Opt[PeerId]): Future[Connection] {.async, gcsafe.} =
if self.secureManagers.len <= 0:
raise newException(UpgradeFailedError, "No secure managers registered!")
@ -67,7 +70,7 @@ proc secure*(
# let's avoid duplicating checks but detect if it fails to do it properly
doAssert(secureProtocol.len > 0)
return await secureProtocol[0].secure(conn, true)
return await secureProtocol[0].secure(conn, true, peerId)
proc identify*(
self: Upgrade,

12
libp2p/utils/heartbeat.nim
View File

@ -25,6 +25,14 @@ template heartbeat*(name: string, interval: Duration, body: untyped): untyped =
nextHeartbeat += interval
let now = Moment.now()
if nextHeartbeat < now:
info "Missed heartbeat", heartbeat = name, delay = now - nextHeartbeat
nextHeartbeat = now + interval
let
delay = now - nextHeartbeat
itv = interval
if delay > itv:
info "Missed multiple heartbeats", heartbeat = name,
delay = delay, hinterval = itv
else:
debug "Missed heartbeat", heartbeat = name,
delay = delay, hinterval = itv
nextHeartbeat = now + itv
await sleepAsync(nextHeartbeat - now)

73
libp2p/utils/offsettedseq.nim Normal file
View File

@ -0,0 +1,73 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import sequtils
type
OffsettedSeq*[T] = object
s*: seq[T]
offset*: int
proc initOffsettedSeq*[T](offset: int = 0): OffsettedSeq[T] =
OffsettedSeq[T](s: newSeq[T](), offset: offset)
proc all*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool =
o.s.all(pred)
proc any*[T](o: OffsettedSeq[T], pred: proc (x: T): bool): bool =
o.s.any(pred)
proc apply*[T](o: OffsettedSeq[T], op: proc (x: T)) =
o.s.apply(pred)
proc apply*[T](o: OffsettedSeq[T], op: proc (x: T): T) =
o.s.apply(pred)
proc apply*[T](o: OffsettedSeq[T], op: proc (x: var T)) =
o.s.apply(pred)
func count*[T](o: OffsettedSeq[T], x: T): int =
o.s.count(x)
proc flushIf*[T](o: OffsettedSeq[T], pred: proc (x: T): bool) =
var i = 0
for e in o.s:
if not pred(e): break
i.inc()
if i > 0:
o.s.delete(0..<i)
o.offset.inc(i)
template flushIfIt*(o, pred: untyped) =
var i = 0
for it {.inject.} in o.s:
if not pred: break
i.inc()
if i > 0:
when (NimMajor, NimMinor) < (1, 4):
o.s.delete(0, i - 1)
else:
o.s.delete(0..<i)
o.offset.inc(i)
proc add*[T](o: var OffsettedSeq[T], v: T) =
o.s.add(v)
proc `[]`*[T](o: var OffsettedSeq[T], index: int): var T =
o.s[index - o.offset]
iterator items*[T](o: OffsettedSeq[T]): T =
for e in o.s:
yield e
proc high*[T](o: OffsettedSeq[T]): int =
o.s.high + o.offset
proc low*[T](o: OffsettedSeq[T]): int =
o.s.low + o.offset

5
libp2p/varint.nim
View File

@ -58,10 +58,7 @@ type
SomeVarint* = PBSomeVarint | LPSomeVarint
SomeUVarint* = PBSomeUVarint | LPSomeUVarint
template toUleb(x: uint64): uint64 = x
template toUleb(x: uint32): uint32 = x
template toUleb(x: uint16): uint16 = x
template toUleb(x: uint8): uint8 = x
template toUleb[T: uint64|uint32|uint16|uint8|uint](x: T): T = x
func toUleb(x: zint64): uint64 =
let v = cast[uint64](x)

14
libp2p/vbuffer.nim
View File

@ -39,21 +39,9 @@ proc len*(vb: VBuffer): int =
result = len(vb.buffer) - vb.offset
doAssert(result >= 0)
proc isLiteral[T](s: seq[T]): bool {.inline.} =
when defined(gcOrc) or defined(gcArc):
false
else:
type
SeqHeader = object
length, reserved: int
(cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0
proc initVBuffer*(data: seq[byte], offset = 0): VBuffer =
## Initialize VBuffer with shallow copy of ``data``.
if isLiteral(data):
result.buffer = data
else:
shallowCopy(result.buffer, data)
result.buffer = data
result.offset = offset
proc initVBuffer*(data: openArray[byte], offset = 0): VBuffer =

19
mkdocs.yml
View File

@ -3,7 +3,9 @@ site_name: nim-libp2p
repo_url: https://github.com/status-im/nim-libp2p
repo_name: status-im/nim-libp2p
site_url: https://status-im.github.io/nim-libp2p/docs
edit_uri: edit/unstable/examples/
# Can't find a way to point the edit to the .nim instead
# of the .md
edit_uri: ''
docs_dir: examples
@ -18,8 +20,8 @@ markdown_extensions:
- pymdownx.superfences
theme:
logo: https://docs.libp2p.io/images/logo_small.png
favicon: https://docs.libp2p.io/images/logo_small.png
logo: https://libp2p.io/img/logo_small.png
favicon: https://libp2p.io/img/logo_small.png
name: material
features:
- navigation.instant
@ -38,8 +40,13 @@ theme:
name: Switch to light mode
nav:
- Introduction: README.md
- Tutorials:
- 'Part I: Simple connection': tutorial_1_connect.md
- 'Part II: Custom protocol': tutorial_2_customproto.md
- 'Introduction': index.md
- 'Simple connection': tutorial_1_connect.md
- 'Create a custom protocol': tutorial_2_customproto.md
- 'Protobuf': tutorial_3_protobuf.md
- 'GossipSub': tutorial_4_gossipsub.md
- 'Discovery Manager': tutorial_5_discovery.md
- 'Game': tutorial_6_game.md
- 'Circuit Relay': circuitrelay.md
- Reference: '/nim-libp2p/master/libp2p.html'

167
nimble.lock
View File

@ -2,53 +2,68 @@
"version": 1,
"packages": {
"unittest2": {
"version": "0.0.4",
"vcsRevision": "f180f596c88dfd266f746ed6f8dbebce39c824db",
"url": "https://github.com/status-im/nim-unittest2.git",
"version": "0.0.5",
"vcsRevision": "da8398c45cafd5bd7772da1fc96e3924a18d3823",
"url": "https://github.com/status-im/nim-unittest2",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "fa309c41eaf6ef57895b9e603f2620a2f6e11780"
"sha1": "b3f8493a4948989ef3e645a38b23aad77e851e26"
}
},
"testutils": {
"version": "0.5.0",
"vcsRevision": "dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34",
"url": "https://github.com/status-im/nim-testutils",
"downloadMethod": "git",
"dependencies": [
"unittest2"
],
"checksums": {
"sha1": "756d0757c4dd06a068f9d38c7f238576ba5ee897"
}
},
"stew": {
"version": "0.1.0",
"vcsRevision": "6ad35b876fb6ebe0dfee0f697af173acc47906ee",
"url": "https://github.com/status-im/nim-stew.git",
"vcsRevision": "7184d2424dc3945657884646a72715d494917aad",
"url": "https://github.com/status-im/nim-stew",
"downloadMethod": "git",
"dependencies": [],
"dependencies": [
"unittest2"
],
"checksums": {
"sha1": "46d58c4feb457f3241e3347778334e325dce5268"
"sha1": "f3125ed2fd126dfd3edbaea14275abd9fa57d703"
}
},
"bearssl": {
"version": "0.1.5",
"vcsRevision": "ba80e2a0d7ae8aab666cee013e38ff8d33a3e5e7",
"version": "0.2.0",
"vcsRevision": "a647994910904b0103a05db3a5ec1ecfc4d91a88",
"url": "https://github.com/status-im/nim-bearssl",
"downloadMethod": "git",
"dependencies": [
"unittest2"
],
"checksums": {
"sha1": "383abd5becc77bf8e365b780a29d20529e1d9c4c"
"sha1": "d634751df2716ea9975912a2d5d0a090bb6bcfa9"
}
},
"httputils": {
"version": "0.3.0",
"vcsRevision": "689da19e9e9cfff4ced85e2b25c6b2b5598ed079",
"url": "https://github.com/status-im/nim-http-utils.git",
"vcsRevision": "a85bd52ae0a956983ca6b3267c72961d2ec0245f",
"url": "https://github.com/status-im/nim-http-utils",
"downloadMethod": "git",
"dependencies": [
"stew"
"stew",
"unittest2"
],
"checksums": {
"sha1": "4ad3ad68d13c50184180ab4b2eacc0bd7ed2ed44"
"sha1": "92933b21bcd29335f68e377e2b2193fa331e28b3"
}
},
"chronos": {
"version": "3.0.11",
"vcsRevision": "17fed89c99beac5a92d3668d0d3e9b0e4ac13936",
"url": "https://github.com/status-im/nim-chronos.git",
"vcsRevision": "75d030ff71264513fb9701c75a326cd36fcb4692",
"url": "https://github.com/status-im/nim-chronos",
"downloadMethod": "git",
"dependencies": [
"stew",
@ -57,52 +72,27 @@
"unittest2"
],
"checksums": {
"sha1": "f6fffc87571e5f76af2a77c4ebcc0e00909ced4e"
}
},
"metrics": {
"version": "0.0.1",
"vcsRevision": "71e0f0e354e1f4c59e3dc92153989c8b723c3440",
"url": "https://github.com/status-im/nim-metrics",
"downloadMethod": "git",
"dependencies": [
"chronos"
],
"checksums": {
"sha1": "86da251fe532ef2163da30343688ab1c148c0340"
}
},
"testutils": {
"version": "0.4.2",
"vcsRevision": "aa6e5216f4b4ab5aa971cdcdd70e1ec1203cedf2",
"url": "https://github.com/status-im/nim-testutils",
"downloadMethod": "git",
"dependencies": [
"unittest2"
],
"checksums": {
"sha1": "94427e0cce0e0c5841edcd3a6530b4e6b857a3cb"
"sha1": "57a674ba3c1a57a694fa7810d93ceb68f338a861"
}
},
"faststreams": {
"version": "0.3.0",
"vcsRevision": "1b561a9e71b6bdad1c1cdff753418906037e9d09",
"url": "https://github.com/status-im/nim-faststreams.git",
"vcsRevision": "b42daf41d8eb4fbce40add6836bed838f8d85b6f",
"url": "https://github.com/status-im/nim-faststreams",
"downloadMethod": "git",
"dependencies": [
"stew",
"testutils",
"chronos",
"unittest2"
],
"checksums": {
"sha1": "97edf9797924af48566a0af8267203dc21d80c77"
"sha1": "62f7ac8fb200a8ecb9e6c63f5553a7dad66ae613"
}
},
"serialization": {
"version": "0.1.0",
"vcsRevision": "fcd0eadadde0ee000a63df8ab21dc4e9f015a790",
"url": "https://github.com/status-im/nim-serialization.git",
"vcsRevision": "d77417cba6896c26287a68e6a95762e45a1b87e5",
"url": "https://github.com/status-im/nim-serialization",
"downloadMethod": "git",
"dependencies": [
"faststreams",
@ -110,70 +100,72 @@
"stew"
],
"checksums": {
"sha1": "fef59519892cac70cccd81b612085caaa5e3e6cf"
"sha1": "e17244c6654de22254acb9bcf71d8ddbeca8b2aa"
}
},
"metrics": {
"version": "0.0.1",
"vcsRevision": "21e99a2e9d9f80e68bef65c80ef781613005fccb",
"url": "https://github.com/status-im/nim-metrics",
"downloadMethod": "git",
"dependencies": [
"chronos"
],
"checksums": {
"sha1": "ab1c994bbcd6b04f2500f05d8ea4e463f33dd310"
}
},
"nimcrypto": {
"version": "0.5.4",
"vcsRevision": "24e006df85927f64916e60511620583b11403178",
"url": "https://github.com/cheatfate/nimcrypto",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "a4db2105de265930f1578bb7957f49fa39b10d9b"
}
},
"json_serialization": {
"version": "0.1.0",
"vcsRevision": "c5f0e2465e8375dfc7aa0f56ccef67cb680bc6b0",
"url": "https://github.com/status-im/nim-json-serialization.git",
"vcsRevision": "a7d815ed92f200f490c95d3cfd722089cc923ce6",
"url": "https://github.com/status-im/nim-json-serialization",
"downloadMethod": "git",
"dependencies": [
"serialization",
"stew"
],
"checksums": {
"sha1": "d89d79d0679a3a41b350e3ad4be56c0308cc5ec6"
"sha1": "50fc34a992ef3df68a7bee88af096bb8ed42572f"
}
},
"chronicles": {
"version": "0.10.2",
"vcsRevision": "1682096306ddba8185dcfac360a8c3f952d721e4",
"url": "https://github.com/status-im/nim-chronicles.git",
"version": "0.10.3",
"vcsRevision": "32ac8679680ea699f7dbc046e8e0131cac97d41a",
"url": "https://github.com/status-im/nim-chronicles",
"downloadMethod": "git",
"dependencies": [
"testutils",
"json_serialization"
],
"checksums": {
"sha1": "9a5bebb76b0f7d587a31e621d260119279e91c76"
}
},
"asynctest": {
"version": "0.3.1",
"vcsRevision": "5347c59b4b057443a014722aa40800cd8bb95c69",
"url": "https://github.com/status-im/asynctest.git",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "53e0b610d13700296755a4ebe789882cae47a3b9"
}
},
"nimcrypto": {
"version": "0.5.4",
"vcsRevision": "a5742a9a214ac33f91615f3862c7b099aec43b00",
"url": "https://github.com/cheatfate/nimcrypto",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "f76c87707cd4e96355b8bb6ef27e7f8b0aac1e08"
"sha1": "79f09526d4d9b9196dd2f6a75310d71a890c4f88"
}
},
"zlib": {
"version": "0.1.0",
"vcsRevision": "74cdeb54b21bededb5a515d36f608bc1850555a2",
"vcsRevision": "6a6670afba6b97b29b920340e2641978c05ab4d8",
"url": "https://github.com/status-im/nim-zlib",
"downloadMethod": "git",
"dependencies": [
"stew"
],
"checksums": {
"sha1": "01d330dc4c1924e56b1559ee73bc760e526f635c"
"sha1": "2621e46369be2a6846713e8c3d681a5bba3e0325"
}
},
"websock": {
"version": "0.1.0",
"vcsRevision": "73edde4417f7b45003113b7a34212c3ccd95b9fd",
"vcsRevision": "691f069b209d372b1240d5ae1f57fb7bbafeaba7",
"url": "https://github.com/status-im/nim-websock",
"downloadMethod": "git",
"dependencies": [
@ -181,36 +173,35 @@
"httputils",
"chronicles",
"stew",
"asynctest",
"nimcrypto",
"bearssl",
"zlib"
],
"checksums": {
"sha1": "ec2b137543f280298ca48de9ed4461a033ba88d3"
"sha1": "c71edfce064e7c0cadde0e687c6edc0caaf9ec07"
}
},
"dnsclient": {
"version": "0.1.2",
"vcsRevision": "fbb76f8af8a33ab818184a7d4406d9fee20993be",
"url": "https://github.com/ba0f3/dnsclient.nim.git",
"version": "0.3.2",
"vcsRevision": "fcd7443634b950eaea574e5eaa00a628ae029823",
"url": "https://github.com/ba0f3/dnsclient.nim",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "663239a914c814204b30dda6e0902cc0fbd0b8ee"
"sha1": "146aa4a8d512a3a786c5bf54311b79900166d9d7"
}
},
"secp256k1": {
"version": "0.5.2",
"vcsRevision": "5340cf188168d6afcafc8023770d880f067c0b2f",
"url": "https://github.com/status-im/nim-secp256k1.git",
"vcsRevision": "fd173fdff863ce2e211cf64c9a03bc7539fe40b0",
"url": "https://github.com/status-im/nim-secp256k1",
"downloadMethod": "git",
"dependencies": [
"stew",
"nimcrypto"
],
"checksums": {
"sha1": "ae9cbea4487be94a06653ffee075a7f1bd1e231e"
"sha1": "657c79f6f2b1b6da92a9cda81ffc9f95d26443cb"
}
}
}

2
tests/commoninterop.nim
View File

@ -3,7 +3,7 @@ import chronos, chronicles, stew/byteutils
import helpers
import ../libp2p
import ../libp2p/[daemon/daemonapi, varint, transports/wstransport, crypto/crypto]
import ../libp2p/protocols/relay/[relay, client, utils]
import ../libp2p/protocols/connectivity/relay/[relay, client, utils]
type
SwitchCreator = proc(

63
tests/commontransport.nim
View File

@ -1,7 +1,7 @@
{.used.}
import sequtils
import chronos, stew/byteutils
import chronos, stew/[byteutils, results]
import ../libp2p/[stream/connection,
transports/transport,
upgrademngrs/upgrade,
@ -13,36 +13,37 @@ import ./helpers
type TransportProvider* = proc(): Transport {.gcsafe, raises: [Defect].}
proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
suite name & " common tests":
teardown:
checkTrackers()
template commonTransportTest*(prov: TransportProvider, ma1: string, ma2: string = "") =
block:
let transpProvider = prov
asyncTest "can handle local address":
let ma = @[MultiAddress.init(ma).tryGet()]
let transport1 = prov()
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = transpProvider()
await transport1.start(ma)
check transport1.handles(transport1.addrs[0])
await transport1.stop()
asyncTest "e2e: handle observedAddr":
let ma = @[MultiAddress.init(ma).tryGet()]
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = prov()
let transport1 = transpProvider()
await transport1.start(ma)
let transport2 = prov()
let transport2 = transpProvider()
proc acceptHandler() {.async, gcsafe.} =
let conn = await transport1.accept()
check transport1.handles(conn.observedAddr)
if conn.observedAddr.isSome():
check transport1.handles(conn.observedAddr.get())
await conn.close()
let handlerWait = acceptHandler()
let conn = await transport2.dial(transport1.addrs[0])
check transport2.handles(conn.observedAddr)
if conn.observedAddr.isSome():
check transport2.handles(conn.observedAddr.get())
await conn.close() #for some protocols, closing requires actively reading, so we must close here
@ -54,9 +55,9 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
await handlerWait.wait(1.seconds) # when no issues will not wait that long!
asyncTest "e2e: handle write":
let ma = @[MultiAddress.init(ma).tryGet()]
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = prov()
let transport1 = transpProvider()
await transport1.start(ma)
proc acceptHandler() {.async, gcsafe.} =
@ -66,7 +67,7 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
let handlerWait = acceptHandler()
let transport2 = prov()
let transport2 = transpProvider()
let conn = await transport2.dial(transport1.addrs[0])
var msg = newSeq[byte](6)
await conn.readExactly(addr msg[0], 6)
@ -82,8 +83,8 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
await handlerWait.wait(1.seconds) # when no issues will not wait that long!
asyncTest "e2e: handle read":
let ma = @[MultiAddress.init(ma).tryGet()]
let transport1 = prov()
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = transpProvider()
await transport1.start(ma)
proc acceptHandler() {.async, gcsafe.} =
@ -95,7 +96,7 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
let handlerWait = acceptHandler()
let transport2 = prov()
let transport2 = transpProvider()
let conn = await transport2.dial(transport1.addrs[0])
await conn.write("Hello!")
@ -108,12 +109,12 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
transport2.stop()))
asyncTest "e2e: handle dial cancellation":
let ma = @[MultiAddress.init(ma).tryGet()]
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = prov()
let transport1 = transpProvider()
await transport1.start(ma)
let transport2 = prov()
let transport2 = transpProvider()
let cancellation = transport2.dial(transport1.addrs[0])
await cancellation.cancelAndWait()
@ -125,9 +126,9 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
transport2.stop()))
asyncTest "e2e: handle accept cancellation":
let ma = @[MultiAddress.init(ma).tryGet()]
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = prov()
let transport1 = transpProvider()
await transport1.start(ma)
let acceptHandler = transport1.accept()
@ -141,11 +142,11 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
# this randomly locks the Windows CI job
skip()
return
let addrs = @[MultiAddress.init(ma).tryGet(),
MultiAddress.init(ma).tryGet()]
let addrs = @[MultiAddress.init(ma1).tryGet(),
MultiAddress.init(if ma2 == "": ma1 else: ma2).tryGet()]
let transport1 = prov()
let transport1 = transpProvider()
await transport1.start(addrs)
proc acceptHandler() {.async, gcsafe.} =
@ -190,12 +191,12 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
await transport1.stop()
asyncTest "e2e: stopping transport kills connections":
let ma = @[MultiAddress.init(ma).tryGet()]
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = prov()
let transport1 = transpProvider()
await transport1.start(ma)
let transport2 = prov()
let transport2 = transpProvider()
let acceptHandler = transport1.accept()
let conn = await transport2.dial(transport1.addrs[0])
@ -210,8 +211,8 @@ proc commonTransportTest*(name: string, prov: TransportProvider, ma: string) =
check conn.closed()
asyncTest "read or write on closed connection":
let ma = @[MultiAddress.init(ma).tryGet()]
let transport1 = prov()
let ma = @[MultiAddress.init(ma1).tryGet()]
let transport1 = transpProvider()
await transport1.start(ma)
proc acceptHandler() {.async, gcsafe.} =

5
tests/config.nims Normal file
View File

@ -0,0 +1,5 @@
import ../config.nims
--threads:on
--d:metrics
--d:withoutPCRE

21
tests/helpers.nim
View File

@ -105,16 +105,15 @@ proc bridgedConnections*: (Connection, Connection) =
return (connA, connB)
proc checkExpiringInternal(cond: proc(): bool {.raises: [Defect].} ): Future[bool] {.async, gcsafe.} =
{.gcsafe.}:
let start = Moment.now()
while true:
if Moment.now() > (start + chronos.seconds(5)):
return false
elif cond():
return true
else:
await sleepAsync(1.millis)
proc checkExpiringInternal(cond: proc(): bool {.raises: [Defect], gcsafe.} ): Future[bool] {.async, gcsafe.} =
let start = Moment.now()
while true:
if Moment.now() > (start + chronos.seconds(5)):
return false
elif cond():
return true
else:
await sleepAsync(1.millis)
template checkExpiring*(code: untyped): untyped =
checkExpiringInternal(proc(): bool = code)
check await checkExpiringInternal(proc(): bool = code)

1
tests/nim.cfg
View File

@ -1 +0,0 @@
--threads:on

34
tests/pubsub/testfloodsub.nim
View File

@ -351,7 +351,7 @@ suite "FloodSub":
check (await smallNode[0].publish("foo", smallMessage1)) > 0
check (await bigNode[0].publish("foo", smallMessage2)) > 0
check (await checkExpiring(messageReceived == 2)) == true
checkExpiring: messageReceived == 2
check (await smallNode[0].publish("foo", bigMessage)) > 0
check (await bigNode[0].publish("foo", bigMessage)) > 0
@ -362,3 +362,35 @@ suite "FloodSub":
)
await allFuturesThrowing(nodesFut)
asyncTest "FloodSub message size validation 2":
var messageReceived = 0
proc handler(topic: string, data: seq[byte]) {.async, gcsafe.} =
inc(messageReceived)
let
bigNode1 = generateNodes(1, maxMessageSize = 20000000)
bigNode2 = generateNodes(1, maxMessageSize = 20000000)
# start switches
nodesFut = await allFinished(
bigNode1[0].switch.start(),
bigNode2[0].switch.start(),
)
await subscribeNodes(bigNode1 & bigNode2)
bigNode2[0].subscribe("foo", handler)
await waitSub(bigNode1[0], bigNode2[0], "foo")
let bigMessage = newSeq[byte](19000000)
check (await bigNode1[0].publish("foo", bigMessage)) > 0
checkExpiring: messageReceived == 1
await allFuturesThrowing(
bigNode1[0].switch.stop(),
bigNode2[0].switch.stop()
)
await allFuturesThrowing(nodesFut)

5
tests/pubsub/testgossipinternal.nim
View File

@ -22,10 +22,7 @@ proc getPubSubPeer(p: TestGossipSub, peerId: PeerId): PubSubPeer =
proc getConn(): Future[Connection] =
p.switch.dial(peerId, GossipSubCodec)
proc dropConn(peer: PubSubPeer) =
discard # we don't care about it here yet
let pubSubPeer = PubSubPeer.new(peerId, getConn, dropConn, nil, GossipSubCodec, 1024 * 1024)
let pubSubPeer = PubSubPeer.new(peerId, getConn, nil, GossipSubCodec, 1024 * 1024)
debug "created new pubsub peer", peerId
p.peers[peerId] = pubSubPeer

69
tests/pubsub/testgossipsub.nim
View File

@ -9,7 +9,7 @@
{.used.}
import sequtils, options, tables, sets
import sequtils, options, tables, sets, sugar
import chronos, stew/byteutils
import chronicles
import utils, ../../libp2p/[errors,
@ -29,46 +29,15 @@ import ../helpers
proc `$`(peer: PubSubPeer): string = shortLog(peer)
proc waitSub(sender, receiver: auto; key: string) {.async, gcsafe.} =
if sender == receiver:
return
# turn things deterministic
# this is for testing purposes only
# peers can be inside `mesh` and `fanout`, not just `gossipsub`
var ceil = 15
let fsub = GossipSub(sender)
let ev = newAsyncEvent()
fsub.heartbeatEvents.add(ev)
# await first heartbeat
await ev.wait()
ev.clear()
while (not fsub.gossipsub.hasKey(key) or
not fsub.gossipsub.hasPeerId(key, receiver.peerInfo.peerId)) and
(not fsub.mesh.hasKey(key) or
not fsub.mesh.hasPeerId(key, receiver.peerInfo.peerId)) and
(not fsub.fanout.hasKey(key) or
not fsub.fanout.hasPeerId(key , receiver.peerInfo.peerId)):
trace "waitSub sleeping..."
# await more heartbeats
await ev.wait()
ev.clear()
dec ceil
doAssert(ceil > 0, "waitSub timeout!")
template tryPublish(call: untyped, require: int, wait: Duration = 1.seconds, times: int = 10): untyped =
template tryPublish(call: untyped, require: int, wait = 10.milliseconds, timeout = 5.seconds): untyped =
var
limit = times
expiration = Moment.now() + timeout
pubs = 0
while pubs < require and limit > 0:
while pubs < require and Moment.now() < expiration:
pubs = pubs + call
await sleepAsync(wait)
limit.dec()
if limit == 0:
doAssert(false, "Failed to publish!")
doAssert pubs >= require, "Failed to publish!"
suite "GossipSub":
teardown:
@ -343,14 +312,13 @@ suite "GossipSub":
await subscribeNodes(nodes)
nodes[1].subscribe("foobar", handler)
await sleepAsync(10.seconds)
let gossip1 = GossipSub(nodes[0])
let gossip2 = GossipSub(nodes[1])
check:
"foobar" in gossip2.topics
"foobar" in gossip1.gossipsub
checkExpiring:
"foobar" in gossip2.topics and
"foobar" in gossip1.gossipsub and
gossip1.gossipsub.hasPeerId("foobar", gossip2.peerInfo.peerId)
await allFuturesThrowing(
@ -494,7 +462,7 @@ suite "GossipSub":
nodes[0].unsubscribe("foobar", handler)
let gsNode = GossipSub(nodes[1])
check await checkExpiring(gsNode.mesh.getOrDefault("foobar").len == 0)
checkExpiring: gsNode.mesh.getOrDefault("foobar").len == 0
nodes[0].subscribe("foobar", handler)
@ -613,7 +581,7 @@ suite "GossipSub":
gossip1.seen = TimedCache[MessageId].init()
gossip3.seen = TimedCache[MessageId].init()
let msgId = toSeq(gossip2.validationSeen.keys)[0]
check await checkExpiring(try: gossip2.validationSeen[msgId].len > 0 except: false)
checkExpiring(try: gossip2.validationSeen[msgId].len > 0 except: false)
result = ValidationResult.Accept
bFinished.complete()
@ -701,14 +669,14 @@ suite "GossipSub":
seenFut.complete()
dialer.subscribe("foobar", handler)
await waitSub(nodes[0], dialer, "foobar")
await waitSubGraph(nodes, "foobar")
tryPublish await wait(nodes[0].publish("foobar",
toBytes("from node " &
$nodes[0].peerInfo.peerId)),
1.minutes), 1, 5.seconds
1.minutes), 1
await wait(seenFut, 2.minutes)
await wait(seenFut, 1.minutes)
check: seen.len >= runs
for k, v in seen.pairs:
check: v >= 1
@ -737,10 +705,11 @@ suite "GossipSub":
var seen: Table[string, int]
var seenFut = newFuture[void]()
for i in 0..<nodes.len:
let dialer = nodes[i]
var handler: TopicHandler
closureScope:
capture dialer, i:
var peerName = $dialer.peerInfo.peerId
handler = proc(topic: string, data: seq[byte]) {.async, gcsafe, closure.} =
if peerName notin seen:
@ -751,14 +720,14 @@ suite "GossipSub":
seenFut.complete()
dialer.subscribe("foobar", handler)
await waitSub(nodes[0], dialer, "foobar")
await waitSubGraph(nodes, "foobar")
tryPublish await wait(nodes[0].publish("foobar",
toBytes("from node " &
$nodes[0].peerInfo.peerId)),
1.minutes), 1, 5.seconds
1.minutes), 1
await wait(seenFut, 5.minutes)
await wait(seenFut, 60.seconds)
check: seen.len >= runs
for k, v in seen.pairs:
check: v >= 1

67
tests/pubsub/testgossipsub2.nim
View File

@ -10,8 +10,7 @@
{.used.}
import sequtils, options, tables, sets
import chronos, stew/byteutils
import chronicles
import chronos, stew/byteutils, chronicles
import utils, ../../libp2p/[errors,
peerid,
peerinfo,
@ -25,46 +24,15 @@ import utils, ../../libp2p/[errors,
protocols/pubsub/rpc/messages]
import ../helpers
proc waitSub(sender, receiver: auto; key: string) {.async, gcsafe.} =
if sender == receiver:
return
# turn things deterministic
# this is for testing purposes only
# peers can be inside `mesh` and `fanout`, not just `gossipsub`
var ceil = 15
let fsub = GossipSub(sender)
let ev = newAsyncEvent()
fsub.heartbeatEvents.add(ev)
# await first heartbeat
await ev.wait()
ev.clear()
while (not fsub.gossipsub.hasKey(key) or
not fsub.gossipsub.hasPeerId(key, receiver.peerInfo.peerId)) and
(not fsub.mesh.hasKey(key) or
not fsub.mesh.hasPeerId(key, receiver.peerInfo.peerId)) and
(not fsub.fanout.hasKey(key) or
not fsub.fanout.hasPeerId(key , receiver.peerInfo.peerId)):
trace "waitSub sleeping..."
# await more heartbeats
await ev.wait()
ev.clear()
dec ceil
doAssert(ceil > 0, "waitSub timeout!")
template tryPublish(call: untyped, require: int, wait: Duration = 1.seconds, times: int = 10): untyped =
template tryPublish(call: untyped, require: int, wait = 10.milliseconds, timeout = 10.seconds): untyped =
var
limit = times
expiration = Moment.now() + timeout
pubs = 0
while pubs < require and limit > 0:
while pubs < require and Moment.now() < expiration:
pubs = pubs + call
await sleepAsync(wait)
limit.dec()
if limit == 0:
doAssert(false, "Failed to publish!")
doAssert pubs >= require, "Failed to publish!"
suite "GossipSub":
teardown:
@ -280,7 +248,7 @@ suite "GossipSub":
await allFuturesThrowing(nodesFut.concat())
asyncTest "GossipsSub peers disconnections mechanics":
asyncTest "GossipSub peers disconnections mechanics":
var runs = 10
let
@ -305,7 +273,8 @@ suite "GossipSub":
seenFut.complete()
dialer.subscribe("foobar", handler)
await waitSub(nodes[0], dialer, "foobar")
await waitSubGraph(nodes, "foobar")
# ensure peer stats are stored properly and kept properly
check:
@ -314,7 +283,7 @@ suite "GossipSub":
tryPublish await wait(nodes[0].publish("foobar",
toBytes("from node " &
$nodes[0].peerInfo.peerId)),
1.minutes), 1, 5.seconds
1.minutes), 1, 5.seconds, 3.minutes
await wait(seenFut, 5.minutes)
check: seen.len >= runs
@ -337,11 +306,9 @@ suite "GossipSub":
# Waiting 2 heartbeats
for _ in 0..1:
for i in 0..<runs:
if i mod 3 == 0:
let evnt = newAsyncEvent()
GossipSub(nodes[i]).heartbeatEvents &= evnt
await evnt.wait()
let evnt = newAsyncEvent()
GossipSub(nodes[0]).heartbeatEvents &= evnt
await evnt.wait()
# ensure peer stats are stored properly and kept properly
check:
@ -359,11 +326,9 @@ suite "GossipSub":
# Waiting 2 heartbeats
for _ in 0..1:
for i in 0..<runs:
if i mod 3 == 0:
let evnt = newAsyncEvent()
GossipSub(nodes[i]).heartbeatEvents &= evnt
await evnt.wait()
let evnt = newAsyncEvent()
GossipSub(nodes[0]).heartbeatEvents &= evnt
await evnt.wait()
# ensure peer stats are stored properly and kept properly
check:

43
tests/pubsub/utils.nim
View File

@ -4,7 +4,7 @@ const
libp2p_pubsub_verify {.booldefine.} = true
libp2p_pubsub_anonymize {.booldefine.} = false
import hashes, random, tables
import hashes, random, tables, sets, sequtils
import chronos, stew/[byteutils, results]
import ../../libp2p/[builders,
protocols/pubsub/errors,
@ -13,6 +13,7 @@ import ../../libp2p/[builders,
protocols/pubsub/floodsub,
protocols/pubsub/rpc/messages,
protocols/secure/secure]
import chronicles
export builders
@ -102,3 +103,43 @@ proc subscribeRandom*(nodes: seq[PubSub]) {.async.} =
if dialer.peerInfo.peerId != node.peerInfo.peerId:
await dialer.switch.connect(node.peerInfo.peerId, node.peerInfo.addrs)
dialed.add(node.peerInfo.peerId)
proc waitSub*(sender, receiver: auto; key: string) {.async, gcsafe.} =
if sender == receiver:
return
let timeout = Moment.now() + 5.seconds
let fsub = GossipSub(sender)
# this is for testing purposes only
# peers can be inside `mesh` and `fanout`, not just `gossipsub`
while (not fsub.gossipsub.hasKey(key) or
not fsub.gossipsub.hasPeerId(key, receiver.peerInfo.peerId)) and
(not fsub.mesh.hasKey(key) or
not fsub.mesh.hasPeerId(key, receiver.peerInfo.peerId)) and
(not fsub.fanout.hasKey(key) or
not fsub.fanout.hasPeerId(key , receiver.peerInfo.peerId)):
trace "waitSub sleeping..."
# await
await sleepAsync(5.milliseconds)
doAssert Moment.now() < timeout, "waitSub timeout!"
proc waitSubGraph*(nodes: seq[PubSub], key: string) {.async, gcsafe.} =
let timeout = Moment.now() + 5.seconds
while true:
var
nodesMesh: Table[PeerId, seq[PeerId]]
seen: HashSet[PeerId]
for n in nodes:
nodesMesh[n.peerInfo.peerId] = toSeq(GossipSub(n).mesh.getOrDefault(key).items()).mapIt(it.peerId)
proc explore(p: PeerId) =
if p in seen: return
seen.incl(p)
for peer in nodesMesh.getOrDefault(p):
explore(peer)
explore(nodes[0].peerInfo.peerId)
if seen.len == nodes.len: return
trace "waitSubGraph sleeping..."
await sleepAsync(5.milliseconds)
doAssert Moment.now() < timeout, "waitSubGraph timeout!"

44
tests/stubs/autonatstub.nim Normal file
View File

@ -0,0 +1,44 @@
{.used.}
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import chronos
import ../../libp2p/protocols/connectivity/autonat
import ../../libp2p/peerid
import ../../libp2p/multiaddress
type
AutonatStub* = ref object of Autonat
answer*: Answer
dials: int
expectedDials: int
finished*: Future[void]
Answer* = enum
Reachable,
NotReachable,
Unknown
proc new*(T: typedesc[AutonatStub], expectedDials: int): T =
return T(dials: 0, expectedDials: expectedDials, finished: newFuture[void]())
method dialMe*(
self: AutonatStub,
pid: PeerId,
addrs: seq[MultiAddress] = newSeq[MultiAddress]()):
Future[MultiAddress] {.async.} =
self.dials += 1
if self.dials == self.expectedDials:
self.finished.complete()
case self.answer:
of Reachable:
return MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
of NotReachable:
raise newException(AutonatUnreachableError, "")
of Unknown:
raise newException(AutonatError, "")

86
tests/stubs/torstub.nim Normal file
View File

@ -0,0 +1,86 @@
{.used.}
when (NimMajor, NimMinor) < (1, 4):
{.push raises: [Defect].}
else:
{.push raises: [].}
import tables
import chronos, stew/[byteutils, endians2, shims/net]
import ../../libp2p/[stream/connection,
protocols/connectivity/relay/utils,
transports/tcptransport,
transports/tortransport,
upgrademngrs/upgrade,
multiaddress,
errors,
builders]
type
TorServerStub* = ref object of RootObj
tcpTransport: TcpTransport
addrTable: Table[string, string]
proc new*(
T: typedesc[TorServerStub]): T {.public.} =
T(
tcpTransport: TcpTransport.new(flags = {ReuseAddr}, upgrade = Upgrade()),
addrTable: initTable[string, string]())
proc registerAddr*(self: TorServerStub, key: string, val: string) =
self.addrTable[key] = val
proc start*(self: TorServerStub, address: TransportAddress) {.async.} =
let ma = @[MultiAddress.init(address).tryGet()]
await self.tcpTransport.start(ma)
var msg = newSeq[byte](3)
while self.tcpTransport.running:
let connSrc = await self.tcpTransport.accept()
await connSrc.readExactly(addr msg[0], 3)
await connSrc.write(@[05'u8, 00])
msg = newSeq[byte](4)
await connSrc.readExactly(addr msg[0], 4)
let atyp = msg[3]
let address = case atyp:
of Socks5AddressType.IPv4.byte:
let n = 4 + 2 # +2 bytes for the port
msg = newSeq[byte](n)
await connSrc.readExactly(addr msg[0], n)
var ip: array[4, byte]
for i, e in msg[0..^3]:
ip[i] = e
$(ipv4(ip)) & ":" & $(Port(fromBytesBE(uint16, msg[^2..^1])))
of Socks5AddressType.IPv6.byte:
let n = 16 + 2 # +2 bytes for the port
msg = newSeq[byte](n) # +2 bytes for the port
await connSrc.readExactly(addr msg[0], n)
var ip: array[16, byte]
for i, e in msg[0..^3]:
ip[i] = e
$(ipv6(ip)) & ":" & $(Port(fromBytesBE(uint16, msg[^2..^1])))
of Socks5AddressType.FQDN.byte:
await connSrc.readExactly(addr msg[0], 1)
let n = int(uint8.fromBytes(msg[0..0])) + 2 # +2 bytes for the port
msg = newSeq[byte](n)
await connSrc.readExactly(addr msg[0], n)
string.fromBytes(msg[0..^3]) & ":" & $(Port(fromBytesBE(uint16, msg[^2..^1])))
else:
raise newException(LPError, "Address not supported")
let tcpIpAddr = self.addrTable[$(address)]
await connSrc.write(@[05'u8, 00, 00, 01, 00, 00, 00, 00, 00, 00])
let connDst = await self.tcpTransport.dial("", MultiAddress.init(tcpIpAddr).tryGet())
await bridge(connSrc, connDst)
await allFutures(connSrc.close(), connDst.close())
proc stop*(self: TorServerStub) {.async.} =
await self.tcpTransport.stop()

89
tests/testautonat.nim Normal file
View File

@ -0,0 +1,89 @@
import std/options
import chronos
import
../libp2p/[
transports/tcptransport,
upgrademngrs/upgrade,
builders,
protocols/connectivity/autonat
],
./helpers
proc createAutonatSwitch(): Switch =
result = SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withMplex()
.withAutonat()
.withNoise()
.build()
proc makeAutonatServicePrivate(): Switch =
var autonatProtocol = new LPProtocol
autonatProtocol.handler = proc (conn: Connection, proto: string) {.async, gcsafe.} =
discard await conn.readLp(1024)
await conn.writeLp(AutonatDialResponse(
status: DialError,
text: some("dial failed"),
ma: none(MultiAddress)).encode().buffer)
await conn.close()
autonatProtocol.codec = AutonatCodec
result = newStandardSwitch()
result.mount(autonatProtocol)
suite "Autonat":
teardown:
checkTrackers()
asyncTest "dialMe returns public address":
let
src = newStandardSwitch()
dst = createAutonatSwitch()
await src.start()
await dst.start()
await src.connect(dst.peerInfo.peerId, dst.peerInfo.addrs)
let ma = await Autonat.new(src).dialMe(dst.peerInfo.peerId, dst.peerInfo.addrs)
check ma in src.peerInfo.addrs
await allFutures(src.stop(), dst.stop())
asyncTest "dialMe handles dial error msg":
let
src = newStandardSwitch()
dst = makeAutonatServicePrivate()
await src.start()
await dst.start()
await src.connect(dst.peerInfo.peerId, dst.peerInfo.addrs)
expect AutonatUnreachableError:
discard await Autonat.new(src).dialMe(dst.peerInfo.peerId, dst.peerInfo.addrs)
await allFutures(src.stop(), dst.stop())
asyncTest "Timeout is triggered in autonat handle":
let
src = newStandardSwitch()
dst = newStandardSwitch()
autonat = Autonat.new(dst, dialTimeout = 1.seconds)
doesNothingListener = TcpTransport.new(upgrade = Upgrade())
dst.mount(autonat)
await src.start()
await dst.start()
await doesNothingListener.start(@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()])
await src.connect(dst.peerInfo.peerId, dst.peerInfo.addrs)
let conn = await src.dial(dst.peerInfo.peerId, @[AutonatCodec])
let buffer = AutonatDial(peerInfo: some(AutonatPeerInfo(
id: some(src.peerInfo.peerId),
# we ask to be dialed in the does nothing listener instead
addrs: doesNothingListener.addrs
))).encode().buffer
await conn.writeLp(buffer)
let response = AutonatMsg.decode(await conn.readLp(1024)).get().response.get()
check:
response.status == DialError
response.text.get() == "Timeout exceeded!"
response.ma.isNone()
await allFutures(doesNothingListener.stop(), src.stop(), dst.stop())

251
tests/testautonatservice.nim Normal file
View File

@ -0,0 +1,251 @@
# Nim-LibP2P
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/options
import chronos, metrics
import unittest2
import ../libp2p/[builders,
switch,
services/autonatservice,
protocols/connectivity/autonat]
import ./helpers
import stubs/autonatstub
proc createSwitch(autonatSvc: Service = nil, withAutonat = true): Switch =
var builder = SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withMplex()
.withNoise()
if withAutonat:
builder = builder.withAutonat()
if autonatSvc != nil:
builder = builder.withServices(@[autonatSvc])
return builder.build()
suite "Autonat Service":
teardown:
checkTrackers()
asyncTest "Peer must be not reachable":
let autonatStub = AutonatStub.new(expectedDials = 3)
autonatStub.answer = NotReachable
let autonatService = AutonatService.new(autonatStub, newRng())
let switch1 = createSwitch(autonatService)
let switch2 = createSwitch()
let switch3 = createSwitch()
let switch4 = createSwitch()
check autonatService.networkReachability() == NetworkReachability.Unknown
await switch1.start()
await switch2.start()
await switch3.start()
await switch4.start()
await switch1.connect(switch2.peerInfo.peerId, switch2.peerInfo.addrs)
await switch1.connect(switch3.peerInfo.peerId, switch3.peerInfo.addrs)
await switch1.connect(switch4.peerInfo.peerId, switch4.peerInfo.addrs)
await autonatStub.finished
check autonatService.networkReachability() == NetworkReachability.NotReachable
check libp2p_autonat_reachability_confidence.value(["NotReachable"]) == 0.3
await allFuturesThrowing(
switch1.stop(), switch2.stop(), switch3.stop(), switch4.stop())
asyncTest "Peer must be reachable":
let autonat = Autonat.new(switch = nil)
let autonatService = AutonatService.new(autonat, newRng(), some(1.seconds))
let switch1 = createSwitch(autonatService)
let switch2 = createSwitch()
let switch3 = createSwitch()
let switch4 = createSwitch()
let awaiter = newFuture[void]()
proc statusAndConfidenceHandler(networkReachability: NetworkReachability, confidence: Option[float]) {.gcsafe, async.} =
if networkReachability == NetworkReachability.Reachable and confidence.isSome() and confidence.get() >= 0.3:
if not awaiter.finished:
awaiter.complete()
autonat.switch = switch1
check autonatService.networkReachability() == NetworkReachability.Unknown
autonatService.statusAndConfidenceHandler(statusAndConfidenceHandler)
await switch1.start()
await switch2.start()
await switch3.start()
await switch4.start()
await switch1.connect(switch2.peerInfo.peerId, switch2.peerInfo.addrs)
await switch1.connect(switch3.peerInfo.peerId, switch3.peerInfo.addrs)
await switch1.connect(switch4.peerInfo.peerId, switch4.peerInfo.addrs)
await awaiter
check autonatService.networkReachability() == NetworkReachability.Reachable
check libp2p_autonat_reachability_confidence.value(["Reachable"]) == 0.3
await allFuturesThrowing(
switch1.stop(), switch2.stop(), switch3.stop(), switch4.stop())
asyncTest "Peer must be not reachable and then reachable":
let autonatStub = AutonatStub.new(expectedDials = 6)
autonatStub.answer = NotReachable
let autonatService = AutonatService.new(autonatStub, newRng(), some(1.seconds))
let switch1 = createSwitch(autonatService)
let switch2 = createSwitch()
let switch3 = createSwitch()
let switch4 = createSwitch()
let awaiter = newFuture[void]()
proc statusAndConfidenceHandler(networkReachability: NetworkReachability, confidence: Option[float]) {.gcsafe, async.} =
if networkReachability == NetworkReachability.NotReachable and confidence.isSome() and confidence.get() >= 0.3:
if not awaiter.finished:
autonatStub.answer = Reachable
awaiter.complete()
check autonatService.networkReachability() == NetworkReachability.Unknown
autonatService.statusAndConfidenceHandler(statusAndConfidenceHandler)
await switch1.start()
await switch2.start()
await switch3.start()
await switch4.start()
await switch1.connect(switch2.peerInfo.peerId, switch2.peerInfo.addrs)
await switch1.connect(switch3.peerInfo.peerId, switch3.peerInfo.addrs)
await switch1.connect(switch4.peerInfo.peerId, switch4.peerInfo.addrs)
await awaiter
check autonatService.networkReachability() == NetworkReachability.NotReachable
check libp2p_autonat_reachability_confidence.value(["NotReachable"]) == 0.3
await autonatStub.finished
check autonatService.networkReachability() == NetworkReachability.Reachable
check libp2p_autonat_reachability_confidence.value(["Reachable"]) == 0.3
await allFuturesThrowing(switch1.stop(), switch2.stop(), switch3.stop(), switch4.stop())
asyncTest "Peer must be reachable when one connected peer has autonat disabled":
let autonat = Autonat.new(switch = nil)
let autonatService = AutonatService.new(autonat, newRng(), some(1.seconds), maxQueueSize = 2)
let switch1 = createSwitch(autonatService)
let switch2 = createSwitch(withAutonat = false)
let switch3 = createSwitch()
let switch4 = createSwitch()
let awaiter = newFuture[void]()
proc statusAndConfidenceHandler(networkReachability: NetworkReachability, confidence: Option[float]) {.gcsafe, async.} =
if networkReachability == NetworkReachability.Reachable and confidence.isSome() and confidence.get() == 1:
if not awaiter.finished:
awaiter.complete()
autonat.switch = switch1
check autonatService.networkReachability() == NetworkReachability.Unknown
autonatService.statusAndConfidenceHandler(statusAndConfidenceHandler)
await switch1.start()
await switch2.start()
await switch3.start()
await switch4.start()
await switch1.connect(switch2.peerInfo.peerId, switch2.peerInfo.addrs)
await switch1.connect(switch3.peerInfo.peerId, switch3.peerInfo.addrs)
await switch1.connect(switch4.peerInfo.peerId, switch4.peerInfo.addrs)
await awaiter
check autonatService.networkReachability() == NetworkReachability.Reachable
check libp2p_autonat_reachability_confidence.value(["Reachable"]) == 1
await allFuturesThrowing(
switch1.stop(), switch2.stop(), switch3.stop(), switch4.stop())
asyncTest "Unknown answers must be ignored":
let autonatStub = AutonatStub.new(expectedDials = 6)
autonatStub.answer = NotReachable
let autonatService = AutonatService.new(autonatStub, newRng(), some(1.seconds), maxQueueSize = 3)
let switch1 = createSwitch(autonatService)
let switch2 = createSwitch()
let switch3 = createSwitch()
let switch4 = createSwitch()
let awaiter = newFuture[void]()
proc statusAndConfidenceHandler(networkReachability: NetworkReachability, confidence: Option[float]) {.gcsafe, async.} =
if networkReachability == NetworkReachability.NotReachable and confidence.isSome() and confidence.get() >= 0.3:
if not awaiter.finished:
autonatStub.answer = Unknown
awaiter.complete()
check autonatService.networkReachability() == NetworkReachability.Unknown
autonatService.statusAndConfidenceHandler(statusAndConfidenceHandler)
await switch1.start()
await switch2.start()
await switch3.start()
await switch4.start()
await switch1.connect(switch2.peerInfo.peerId, switch2.peerInfo.addrs)
await switch1.connect(switch3.peerInfo.peerId, switch3.peerInfo.addrs)
await switch1.connect(switch4.peerInfo.peerId, switch4.peerInfo.addrs)
await awaiter
check autonatService.networkReachability() == NetworkReachability.NotReachable
check libp2p_autonat_reachability_confidence.value(["NotReachable"]) == 1/3
await autonatStub.finished
check autonatService.networkReachability() == NetworkReachability.NotReachable
check libp2p_autonat_reachability_confidence.value(["NotReachable"]) == 1/3
await allFuturesThrowing(switch1.stop(), switch2.stop(), switch3.stop(), switch4.stop())
asyncTest "Calling setup and stop twice must work":
let switch = createSwitch()
let autonatService = AutonatService.new(AutonatStub.new(expectedDials = 0), newRng(), some(1.seconds))
check (await autonatService.setup(switch)) == true
check (await autonatService.setup(switch)) == false
check (await autonatService.stop(switch)) == true
check (await autonatService.stop(switch)) == false
await allFuturesThrowing(switch.stop())

38
tests/testconnmngr.nim
View File

@ -1,4 +1,5 @@
import sequtils
import stew/results
import chronos
import ../libp2p/[connmanager,
stream/connection,
@ -9,6 +10,9 @@ import ../libp2p/[connmanager,
import helpers
proc getConnection(peerId: PeerId, dir: Direction = Direction.In): Connection =
return Connection.new(peerId, dir, Opt.none(MultiAddress))
type
TestMuxer = ref object of Muxer
peerId: PeerId
@ -18,7 +22,7 @@ method newStream*(
name: string = "",
lazy: bool = false):
Future[Connection] {.async, gcsafe.} =
result = Connection.new(m.peerId, Direction.Out)
result = getConnection(m.peerId, Direction.Out)
suite "Connection Manager":
teardown:
@ -27,7 +31,7 @@ suite "Connection Manager":
asyncTest "add and retrieve a connection":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
connMngr.storeConn(conn)
check conn in connMngr
@ -41,7 +45,7 @@ suite "Connection Manager":
asyncTest "shouldn't allow a closed connection":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
await conn.close()
expect CatchableError:
@ -52,7 +56,7 @@ suite "Connection Manager":
asyncTest "shouldn't allow an EOFed connection":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
conn.isEof = true
expect CatchableError:
@ -64,7 +68,7 @@ suite "Connection Manager":
asyncTest "add and retrieve a muxer":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
let muxer = new Muxer
muxer.connection = conn
@ -80,7 +84,7 @@ suite "Connection Manager":
asyncTest "shouldn't allow a muxer for an untracked connection":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
let muxer = new Muxer
muxer.connection = conn
@ -94,8 +98,8 @@ suite "Connection Manager":
asyncTest "get conn with direction":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn1 = Connection.new(peerId, Direction.Out)
let conn2 = Connection.new(peerId, Direction.In)
let conn1 = getConnection(peerId, Direction.Out)
let conn2 = getConnection(peerId)
connMngr.storeConn(conn1)
connMngr.storeConn(conn2)
@ -114,7 +118,7 @@ suite "Connection Manager":
asyncTest "get muxed stream for peer":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
let muxer = new TestMuxer
muxer.peerId = peerId
@ -134,7 +138,7 @@ suite "Connection Manager":
asyncTest "get stream from directed connection":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
let muxer = new TestMuxer
muxer.peerId = peerId
@ -155,7 +159,7 @@ suite "Connection Manager":
asyncTest "get stream from any connection":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
let muxer = new TestMuxer
muxer.peerId = peerId
@ -175,11 +179,11 @@ suite "Connection Manager":
let connMngr = ConnManager.new(maxConnsPerPeer = 1)
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
connMngr.storeConn(Connection.new(peerId, Direction.In))
connMngr.storeConn(getConnection(peerId))
let conns = @[
Connection.new(peerId, Direction.In),
Connection.new(peerId, Direction.In)]
getConnection(peerId),
getConnection(peerId)]
expect TooManyConnectionsError:
connMngr.storeConn(conns[0])
@ -193,7 +197,7 @@ suite "Connection Manager":
asyncTest "cleanup on connection close":
let connMngr = ConnManager.new()
let peerId = PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet()
let conn = Connection.new(peerId, Direction.In)
let conn = getConnection(peerId)
let muxer = new Muxer
muxer.connection = conn
@ -220,7 +224,7 @@ suite "Connection Manager":
Direction.In else:
Direction.Out
let conn = Connection.new(peerId, dir)
let conn = getConnection(peerId, dir)
let muxer = new Muxer
muxer.connection = conn
@ -353,7 +357,7 @@ suite "Connection Manager":
let slot = await ((connMngr.getOutgoingSlot()).wait(10.millis))
let conn =
Connection.new(
getConnection(
PeerId.init(PrivateKey.random(ECDSA, (newRng())[]).tryGet()).tryGet(),
Direction.In)

Some files were not shown because too many files have changed in this diff Show More