mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-02 13:33:10 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
47061bf29b | ||
|
|
7ba5e8c13a | ||
|
|
484124db09 | ||
|
|
89917d4bb6 | ||
|
|
7602adc0df | ||
|
|
15ff87a8bb |
@ -78,6 +78,12 @@ runs:
|
||||
mingw-w64-i686-ntldd-git
|
||||
mingw-w64-i686-rust
|
||||
|
||||
- name: MSYS2 (Windows All) - Downgrade to gcc 13
|
||||
if: inputs.os == 'windows'
|
||||
shell: ${{ inputs.shell }} {0}
|
||||
run: |
|
||||
pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-13.2.0-6-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-13.2.0-6-any.pkg.tar.zst
|
||||
|
||||
- name: Derive environment variables
|
||||
shell: ${{ inputs.shell }} {0}
|
||||
run: |
|
||||
|
||||
9
.github/workflows/ci-reusable.yml
vendored
9
.github/workflows/ci-reusable.yml
vendored
@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}'
|
||||
runs-on: ${{ matrix.builder }}
|
||||
timeout-minutes: 80
|
||||
timeout-minutes: 100
|
||||
steps:
|
||||
- name: Checkout sources
|
||||
uses: actions/checkout@v4
|
||||
@ -53,7 +53,7 @@ jobs:
|
||||
node-version: 18.15
|
||||
|
||||
- name: Start Ethereum node with Codex contracts
|
||||
if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all'
|
||||
if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
|
||||
working-directory: vendor/codex-contracts-eth
|
||||
env:
|
||||
MSYS2_PATH_TYPE: inherit
|
||||
@ -79,6 +79,11 @@ jobs:
|
||||
path: tests/integration/logs/
|
||||
retention-days: 1
|
||||
|
||||
## Part 4 Tools ##
|
||||
- name: Tools tests
|
||||
if: matrix.tests == 'tools' || matrix.tests == 'all'
|
||||
run: make -j${ncpu} testTools
|
||||
|
||||
status:
|
||||
if: always()
|
||||
needs: [build]
|
||||
|
||||
1
.github/workflows/ci.yml
vendored
1
.github/workflows/ci.yml
vendored
@ -33,6 +33,7 @@ jobs:
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
|
||||
build:
|
||||
needs: matrix
|
||||
|
||||
2
.github/workflows/docker-dist-tests.yml
vendored
2
.github/workflows/docker-dist-tests.yml
vendored
@ -24,7 +24,7 @@ jobs:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
with:
|
||||
nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true'
|
||||
nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true'
|
||||
nat_ip_auto: true
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
tag_suffix: dist-tests
|
||||
|
||||
2
.github/workflows/nim-matrix.yml
vendored
2
.github/workflows/nim-matrix.yml
vendored
@ -6,7 +6,7 @@ on:
|
||||
|
||||
env:
|
||||
cache_nonce: 0 # Allows for easily busting actions/cache caches
|
||||
nim_version: pinned, v1.6.16, v1.6.18
|
||||
nim_version: pinned
|
||||
|
||||
jobs:
|
||||
matrix:
|
||||
|
||||
106
.github/workflows/release.yml
vendored
106
.github/workflows/release.yml
vendored
@ -8,10 +8,13 @@ on:
|
||||
|
||||
env:
|
||||
cache_nonce: 0 # Allows for easily busting actions/cache caches
|
||||
nim_version: v1.6.14
|
||||
nim_version: pinned
|
||||
rust_version: 1.78.0
|
||||
binary_base: codex
|
||||
upload_to_codex: false
|
||||
codex_binary_base: codex
|
||||
cirdl_binary_base: cirdl
|
||||
build_dir: build
|
||||
nim_flags: ''
|
||||
windows_libs: 'libstdc++-6.dll libgomp-1.dll libgcc_s_seh-1.dll libwinpthread-1.dll'
|
||||
|
||||
jobs:
|
||||
# Matrix
|
||||
@ -25,7 +28,7 @@ jobs:
|
||||
uses: fabiocaccamo/create-matrix-action@v4
|
||||
with:
|
||||
matrix: |
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-22.04}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-20.04}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {linux}, cpu {arm64}, builder {buildjet-4vcpu-ubuntu-2204-arm}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {amd64}, builder {macos-13}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {arm64}, builder {macos-14}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
@ -68,20 +71,50 @@ jobs:
|
||||
macos*) os_name="darwin" ;;
|
||||
windows*) os_name="windows" ;;
|
||||
esac
|
||||
binary="${{ env.binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
|
||||
[[ ${os_name} == "windows" ]] && binary="${binary}.exe"
|
||||
echo "binary=${binary}" >>$GITHUB_ENV
|
||||
github_ref_name="${GITHUB_REF_NAME/\//-}"
|
||||
codex_binary="${{ env.codex_binary_base }}-${github_ref_name}-${os_name}-${{ matrix.cpu }}"
|
||||
cirdl_binary="${{ env.cirdl_binary_base }}-${github_ref_name}-${os_name}-${{ matrix.cpu }}"
|
||||
if [[ ${os_name} == "windows" ]]; then
|
||||
codex_binary="${codex_binary}.exe"
|
||||
cirdl_binary="${cirdl_binary}.exe"
|
||||
fi
|
||||
echo "codex_binary=${codex_binary}" >>$GITHUB_ENV
|
||||
echo "cirdl_binary=${cirdl_binary}" >>$GITHUB_ENV
|
||||
|
||||
- name: Release - Build
|
||||
run: |
|
||||
make NIMFLAGS="--out:${{ env.binary }}"
|
||||
|
||||
- name: Release - Upload binaries
|
||||
make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.codex_binary }} ${{ env.nim_flags }}"
|
||||
make cirdl NIMFLAGS="--out:${{ env.build_dir }}/${{ env.cirdl_binary }} ${{ env.nim_flags }}"
|
||||
|
||||
- name: Release - Libraries
|
||||
run: |
|
||||
if [[ "${{ matrix.os }}" == "windows" ]]; then
|
||||
for lib in ${{ env.windows_libs }}; do
|
||||
cp -v "${MINGW_PREFIX}/bin/${lib}" "${{ env.build_dir }}"
|
||||
done
|
||||
fi
|
||||
|
||||
- name: Release - Upload codex build artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: release-${{ env.binary }}
|
||||
path: ${{ env.binary }}
|
||||
retention-days: 1
|
||||
name: release-${{ env.codex_binary }}
|
||||
path: ${{ env.build_dir }}/${{ env.codex_binary_base }}*
|
||||
retention-days: 30
|
||||
|
||||
- name: Release - Upload cirdl build artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: release-${{ env.cirdl_binary }}
|
||||
path: ${{ env.build_dir }}/${{ env.cirdl_binary_base }}*
|
||||
retention-days: 30
|
||||
|
||||
- name: Release - Upload windows libs
|
||||
if: matrix.os == 'windows'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: release-${{ matrix.os }}-libs
|
||||
path: ${{ env.build_dir }}/*.dll
|
||||
retention-days: 30
|
||||
|
||||
# Release
|
||||
release:
|
||||
@ -99,46 +132,47 @@ jobs:
|
||||
- name: Release - Compress and checksum
|
||||
run: |
|
||||
cd /tmp/release
|
||||
prepare() {
|
||||
# Checksum
|
||||
checksum() {
|
||||
arc="${1}"
|
||||
sha256sum "${arc}" >"${arc}.sha256"
|
||||
|
||||
# Upload to Codex
|
||||
if [[ "${{ env.upload_to_codex }}" == "true" ]]; then
|
||||
codex_endpoints="${{ secrets.CODEX_ENDPOINTS }}"
|
||||
codex_username="${{ secrets.CODEX_USERNAME }}"
|
||||
codex_password="${{ secrets.CODEX_PASSWORD }}"
|
||||
|
||||
for endpoint in ${codex_endpoints}; do
|
||||
echo "::add-mask::${endpoint}"
|
||||
cid=$(curl -X POST \
|
||||
"${endpoint}/api/codex/v1/data" \
|
||||
-u "${codex_username}":"${codex_password}" \
|
||||
-H "content-type: application/octet-stream" \
|
||||
-T "${arc}")
|
||||
|
||||
echo "${cid}" >"${arc}.cid"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# Compress and prepare
|
||||
for file in *; do
|
||||
for file in ${{ env.codex_binary_base }}* ${{ env.cirdl_binary_base }}*; do
|
||||
if [[ "${file}" == *".exe"* ]]; then
|
||||
|
||||
# Windows - binary only
|
||||
arc="${file%.*}.zip"
|
||||
zip "${arc}" "${file}"
|
||||
checksum "${arc}"
|
||||
|
||||
# Windows - binary and libs
|
||||
arc="${file%.*}-libs.zip"
|
||||
zip "${arc}" "${file}" ${{ env.windows_libs }}
|
||||
rm -f "${file}"
|
||||
prepare "${arc}"
|
||||
checksum "${arc}"
|
||||
else
|
||||
|
||||
# Linux/macOS
|
||||
arc="${file}.tar.gz"
|
||||
chmod 755 "${file}"
|
||||
tar cfz "${arc}" "${file}"
|
||||
rm -f "${file}"
|
||||
prepare "${arc}"
|
||||
checksum "${arc}"
|
||||
fi
|
||||
done
|
||||
rm -f ${{ env.windows_libs }}
|
||||
|
||||
- name: Release - Upload compressed artifacts and checksums
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: archives-and-checksums
|
||||
path: /tmp/release/
|
||||
retention-days: 30
|
||||
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
files: |
|
||||
/tmp/release/*
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -3,6 +3,7 @@
|
||||
!*.*
|
||||
*.exe
|
||||
|
||||
!LICENSE*
|
||||
!Makefile
|
||||
|
||||
nimcache/
|
||||
|
||||
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -215,3 +215,6 @@
|
||||
[submodule "vendor/nim-leveldbstatic"]
|
||||
path = vendor/nim-leveldbstatic
|
||||
url = https://github.com/codex-storage/nim-leveldb.git
|
||||
[submodule "vendor/nim-zippy"]
|
||||
path = vendor/nim-zippy
|
||||
url = https://github.com/status-im/nim-zippy.git
|
||||
|
||||
192
BUILDING.md
192
BUILDING.md
@ -1,192 +0,0 @@
|
||||
# Building Codex
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Install developer tools](#prerequisites)
|
||||
- [Linux](#linux)
|
||||
- [macOS](#macos)
|
||||
- [Windows + MSYS2](#windows--msys2)
|
||||
- [Other](#other)
|
||||
- [Clone and prepare the Git repository](#repository)
|
||||
- [Build the executable](#executable)
|
||||
- [Run the example](#example-usage)
|
||||
|
||||
**Optional**
|
||||
- [Run the tests](#tests)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
To build nim-codex, developer tools need to be installed and accessible in the OS.
|
||||
|
||||
Instructions below correspond roughly to environmental setups in nim-codex's [CI workflow](https://github.com/codex-storage/nim-codex/blob/main/.github/workflows/ci.yml) and are known to work.
|
||||
|
||||
Other approaches may be viable. On macOS, some users may prefer [MacPorts](https://www.macports.org/) to [Homebrew](https://brew.sh/). On Windows, rather than use MSYS2, some users may prefer to install developer tools with [winget](https://docs.microsoft.com/en-us/windows/package-manager/winget/), [Scoop](https://scoop.sh/), or [Chocolatey](https://chocolatey.org/), or download installers for e.g. Make and CMake while otherwise relying on official Windows developer tools. Community contributions to these docs and our build system are welcome!
|
||||
|
||||
### Rust
|
||||
|
||||
The current implementation of Codex's zero-knowledge proving circuit requires the installation of rust v1.76.0 or greater. Be sure to install it for your OS and add it to your terminal's path such that the command `cargo --version` gives a compatible version.
|
||||
|
||||
### Linux
|
||||
|
||||
*Package manager commands may require `sudo` depending on OS setup.*
|
||||
|
||||
On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run
|
||||
|
||||
```shell
|
||||
apt-get update && apt-get install build-essential cmake curl git rustc cargo
|
||||
```
|
||||
|
||||
Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc.
|
||||
|
||||
For example, on a bare bones installation of Fedora, run
|
||||
|
||||
```shell
|
||||
dnf install @development-tools cmake gcc-c++ rust cargo
|
||||
```
|
||||
|
||||
In case your distribution does not provide required Rust version, we may install it using [rustup](https://www.rust-lang.org/tools/install)
|
||||
```shell
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --default-toolchain=1.76.0 -y
|
||||
|
||||
. "$HOME/.cargo/env"
|
||||
```
|
||||
|
||||
### macOS
|
||||
|
||||
Install the [Xcode Command Line Tools](https://mac.install.guide/commandlinetools/index.html) by opening a terminal and running
|
||||
```shell
|
||||
xcode-select --install
|
||||
```
|
||||
|
||||
Install [Homebrew (`brew`)](https://brew.sh/) and in a new terminal run
|
||||
```shell
|
||||
brew install bash cmake rust
|
||||
```
|
||||
|
||||
Check that `PATH` is setup correctly
|
||||
```shell
|
||||
which bash cmake
|
||||
|
||||
# /usr/local/bin/bash
|
||||
# /usr/local/bin/cmake
|
||||
```
|
||||
|
||||
### Windows + MSYS2
|
||||
|
||||
*Instructions below assume the OS is 64-bit Windows and that the hardware or VM is [x86-64](https://en.wikipedia.org/wiki/X86-64) compatible.*
|
||||
|
||||
Download and run the installer from [msys2.org](https://www.msys2.org/).
|
||||
|
||||
Launch an MSYS2 [environment](https://www.msys2.org/docs/environments/). UCRT64 is generally recommended: from the Windows *Start menu* select `MSYS2 MinGW UCRT x64`.
|
||||
|
||||
Assuming a UCRT64 environment, in Bash run
|
||||
```shell
|
||||
pacman -Suy
|
||||
pacman -S base-devel git unzip mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-rust
|
||||
```
|
||||
|
||||
<!-- #### Headless Windows container -->
|
||||
<!-- add instructions re: getting setup with MSYS2 in a Windows container -->
|
||||
<!-- https://github.com/StefanScherer/windows-docker-machine -->
|
||||
|
||||
#### Optional: VSCode Terminal integration
|
||||
|
||||
You can link the MSYS2-UCRT64 terminal into VSCode by modifying the configuration file as shown below.
|
||||
File: `C:/Users/<username>/AppData/Roaming/Code/User/settings.json`
|
||||
```json
|
||||
{
|
||||
...
|
||||
"terminal.integrated.profiles.windows": {
|
||||
...
|
||||
"MSYS2-UCRT64": {
|
||||
"path": "C:\\msys64\\usr\\bin\\bash.exe",
|
||||
"args": [
|
||||
"--login",
|
||||
"-i"
|
||||
],
|
||||
"env": {
|
||||
"MSYSTEM": "UCRT64",
|
||||
"CHERE_INVOKING": "1",
|
||||
"MSYS2_PATH_TYPE": "inherit"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Other
|
||||
|
||||
It is possible that nim-codex can be built and run on other platforms supported by the [Nim](https://nim-lang.org/) language: BSD family, older versions of Windows, etc. There has not been sufficient experimentation with nim-codex on such platforms, so instructions are not provided. Community contributions to these docs and our build system are welcome!
|
||||
|
||||
## Repository
|
||||
|
||||
In Bash run
|
||||
```shell
|
||||
git clone https://github.com/codex-storage/nim-codex.git repos/nim-codex && cd repos/nim-codex
|
||||
```
|
||||
|
||||
nim-codex uses the [nimbus-build-system](https://github.com/status-im/nimbus-build-system), so next run
|
||||
```shell
|
||||
make update
|
||||
```
|
||||
|
||||
This step can take a while to complete because by default it builds the [Nim compiler](https://nim-lang.org/docs/nimc.html).
|
||||
|
||||
To see more output from `make` pass `V=1`. This works for all `make` targets in projects using the nimbus-build-system
|
||||
```shell
|
||||
make V=1 update
|
||||
```
|
||||
|
||||
## Executable
|
||||
|
||||
In Bash run
|
||||
```shell
|
||||
make
|
||||
```
|
||||
|
||||
The default `make` target creates the `build/codex` executable.
|
||||
|
||||
## Example usage
|
||||
|
||||
See the [instructions](README.md#cli-options) in the main readme.
|
||||
|
||||
## Tests
|
||||
|
||||
In Bash run
|
||||
```shell
|
||||
make test
|
||||
```
|
||||
|
||||
### testAll
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
To run the integration tests, an Ethereum test node is required. Follow these instructions to set it up.
|
||||
|
||||
##### Windows (do this before 'All platforms')
|
||||
|
||||
1. Download and install Visual Studio 2017 or newer. (Not VSCode!) In the Workloads overview, enable `Desktop development with C++`. ( https://visualstudio.microsoft.com )
|
||||
|
||||
##### All platforms
|
||||
|
||||
1. Install NodeJS (tested with v18.14.0), consider using NVM as a version manager. [Node Version Manager (`nvm`)](https://github.com/nvm-sh/nvm#readme)
|
||||
1. Open a terminal
|
||||
1. Go to the vendor/codex-contracts-eth folder: `cd /<git-root>/vendor/codex-contracts-eth/`
|
||||
1. `npm install` -> Should complete with the number of packages added and an overview of known vulnerabilities.
|
||||
1. `npm test` -> Should output test results. May take a minute.
|
||||
|
||||
Before the integration tests are started, you must start the Ethereum test node manually.
|
||||
1. Open a terminal
|
||||
1. Go to the vendor/codex-contracts-eth folder: `cd /<git-root>/vendor/codex-contracts-eth/`
|
||||
1. `npm start` -> This should launch Hardhat, and output a number of keys and a warning message.
|
||||
|
||||
#### Run
|
||||
|
||||
The `testAll` target runs the same tests as `make test` and also runs tests for nim-codex's Ethereum contracts, as well a basic suite of integration tests.
|
||||
|
||||
To run `make testAll`.
|
||||
|
||||
Use a new terminal to run:
|
||||
```shell
|
||||
make testAll
|
||||
```
|
||||
199
LICENSE-APACHEv2
Normal file
199
LICENSE-APACHEv2
Normal file
@ -0,0 +1,199 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
19
LICENSE-MIT
Normal file
19
LICENSE-MIT
Normal file
@ -0,0 +1,19 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
14
Makefile
14
Makefile
@ -15,7 +15,7 @@
|
||||
#
|
||||
# If NIM_COMMIT is set to "nimbusbuild", this will use the
|
||||
# version pinned by nimbus-build-system.
|
||||
PINNED_NIM_VERSION := v1.6.14
|
||||
PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21
|
||||
|
||||
ifeq ($(NIM_COMMIT),)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
@ -74,6 +74,11 @@ all: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim codex $(NIM_PARAMS) build.nims
|
||||
|
||||
# Build tools/cirdl
|
||||
cirdl: | deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim toolsCirdl $(NIM_PARAMS) build.nims
|
||||
|
||||
# must be included after the default target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
|
||||
|
||||
@ -124,7 +129,12 @@ testAll: | build deps
|
||||
# Builds and runs Taiko L2 tests
|
||||
testTaiko: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) codex.nims
|
||||
$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims
|
||||
|
||||
# Builds and runs tool tests
|
||||
testTools: | cirdl
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims
|
||||
|
||||
# nim-libbacktrace
|
||||
LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
|
||||
|
||||
110
README.md
110
README.md
@ -16,7 +16,7 @@
|
||||
|
||||
## Build and Run
|
||||
|
||||
For detailed instructions on preparing to build nim-codex see [*Building Codex*](BUILDING.md).
|
||||
For detailed instructions on preparing to build nim-codex see [*Build Codex*](https://docs.codex.storage/learn/build).
|
||||
|
||||
To build the project, clone it and run:
|
||||
|
||||
@ -35,112 +35,18 @@ build/codex
|
||||
|
||||
It is possible to configure a Codex node in several ways:
|
||||
1. CLI options
|
||||
2. Env. variable
|
||||
3. Config
|
||||
2. Environment variables
|
||||
3. Configuration file
|
||||
|
||||
The order of priority is the same as above: Cli arguments > Env variables > Config file values.
|
||||
The order of priority is the same as above: CLI options --> Environment variables --> Configuration file.
|
||||
|
||||
### Environment variables
|
||||
Please check [documentation](https://docs.codex.storage/learn/run#configuration) for more information.
|
||||
|
||||
In order to set a configuration option using environment variables, first find the desired CLI option
|
||||
and then transform it in the following way:
|
||||
|
||||
1. prepend it with `CODEX_`
|
||||
2. make it uppercase
|
||||
3. replace `-` with `_`
|
||||
|
||||
For example, to configure `--log-level`, use `CODEX_LOG_LEVEL` as the environment variable name.
|
||||
|
||||
### Configuration file
|
||||
|
||||
A [TOML](https://toml.io/en/) configuration file can also be used to set configuration values. Configuration option names and corresponding values are placed in the file, separated by `=`. Configuration option names can be obtained from the `codex --help` command, and should not include the `--` prefix. For example, a node's log level (`--log-level`) can be configured using TOML as follows:
|
||||
|
||||
```toml
|
||||
log-level = "trace"
|
||||
```
|
||||
|
||||
The Codex node can then read the configuration from this file using the `--config-file` CLI parameter, like `codex --config-file=/path/to/your/config.toml`.
|
||||
|
||||
### CLI Options
|
||||
|
||||
```
|
||||
build/codex --help
|
||||
Usage:
|
||||
|
||||
codex [OPTIONS]... command
|
||||
|
||||
The following options are available:
|
||||
|
||||
--config-file Loads the configuration from a TOML file [=none].
|
||||
--log-level Sets the log level [=info].
|
||||
--metrics Enable the metrics server [=false].
|
||||
--metrics-address Listening address of the metrics server [=127.0.0.1].
|
||||
--metrics-port Listening HTTP port of the metrics server [=8008].
|
||||
-d, --data-dir The directory where codex will store configuration and data.
|
||||
-i, --listen-addrs Multi Addresses to listen on [=/ip4/0.0.0.0/tcp/0].
|
||||
-a, --nat IP Addresses to announce behind a NAT [=127.0.0.1].
|
||||
-e, --disc-ip Discovery listen address [=0.0.0.0].
|
||||
-u, --disc-port Discovery (UDP) port [=8090].
|
||||
--net-privkey Source of network (secp256k1) private key file path or name [=key].
|
||||
-b, --bootstrap-node Specifies one or more bootstrap nodes to use when connecting to the network.
|
||||
--max-peers The maximum number of peers to connect to [=160].
|
||||
--agent-string Node agent string which is used as identifier in network [=Codex].
|
||||
--api-bindaddr The REST API bind address [=127.0.0.1].
|
||||
-p, --api-port The REST Api port [=8080].
|
||||
--repo-kind Backend for main repo store (fs, sqlite) [=fs].
|
||||
-q, --storage-quota The size of the total storage quota dedicated to the node [=8589934592].
|
||||
-t, --block-ttl Default block timeout in seconds - 0 disables the ttl [=$DefaultBlockTtl].
|
||||
--block-mi Time interval in seconds - determines frequency of block maintenance cycle: how
|
||||
often blocks are checked for expiration and cleanup
|
||||
[=$DefaultBlockMaintenanceInterval].
|
||||
--block-mn Number of blocks to check every maintenance cycle [=1000].
|
||||
-c, --cache-size The size of the block cache, 0 disables the cache - might help on slow hardrives
|
||||
[=0].
|
||||
|
||||
Available sub-commands:
|
||||
|
||||
codex persistence [OPTIONS]... command
|
||||
|
||||
The following options are available:
|
||||
|
||||
--eth-provider The URL of the JSON-RPC API of the Ethereum node [=ws://localhost:8545].
|
||||
--eth-account The Ethereum account that is used for storage contracts.
|
||||
--eth-private-key File containing Ethereum private key for storage contracts.
|
||||
--marketplace-address Address of deployed Marketplace contract.
|
||||
--validator Enables validator, requires an Ethereum node [=false].
|
||||
--validator-max-slots Maximum number of slots that the validator monitors [=1000].
|
||||
|
||||
Available sub-commands:
|
||||
|
||||
codex persistence prover [OPTIONS]...
|
||||
|
||||
The following options are available:
|
||||
|
||||
--circom-r1cs The r1cs file for the storage circuit.
|
||||
--circom-wasm The wasm file for the storage circuit.
|
||||
--circom-zkey The zkey file for the storage circuit.
|
||||
--circom-no-zkey Ignore the zkey file - use only for testing! [=false].
|
||||
--proof-samples Number of samples to prove [=5].
|
||||
--max-slot-depth The maximum depth of the slot tree [=32].
|
||||
--max-dataset-depth The maximum depth of the dataset tree [=8].
|
||||
--max-block-depth The maximum depth of the network block merkle tree [=5].
|
||||
--max-cell-elements The maximum number of elements in a cell [=67].
|
||||
```
|
||||
|
||||
#### Logging
|
||||
|
||||
Codex uses [Chronicles](https://github.com/status-im/nim-chronicles) logging library, which allows great flexibility in working with logs.
|
||||
Chronicles has the concept of topics, which categorize log entries into semantic groups.
|
||||
|
||||
Using the `log-level` parameter, you can set the top-level log level like `--log-level="trace"`, but more importantly,
|
||||
you can set log levels for specific topics like `--log-level="info; trace: marketplace,node; error: blockexchange"`,
|
||||
which sets the top-level log level to `info` and then for topics `marketplace` and `node` sets the level to `trace` and so on.
|
||||
|
||||
### Guides
|
||||
## Guides
|
||||
|
||||
To get acquainted with Codex, consider:
|
||||
* running the simple [Codex Two-Client Test](docs/TwoClientTest.md) for a start, and;
|
||||
* if you are feeling more adventurous, try [Running a Local Codex Network with Marketplace Support](docs/Marketplace.md) using a local blockchain as well.
|
||||
* running the simple [Codex Two-Client Test](https://docs.codex.storage/learn/local-two-client-test) for a start, and;
|
||||
* if you are feeling more adventurous, try [Running a Local Codex Network with Marketplace Support](https://docs.codex.storage/learn/local-marketplace) using a local blockchain as well.
|
||||
|
||||
## API
|
||||
|
||||
|
||||
15
build.nims
15
build.nims
@ -1,5 +1,6 @@
|
||||
mode = ScriptMode.Verbose
|
||||
|
||||
import std/os except commandLineParams
|
||||
|
||||
### Helper functions
|
||||
proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
|
||||
@ -14,7 +15,11 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
|
||||
for i in 2..<paramCount():
|
||||
extra_params &= " " & paramStr(i)
|
||||
|
||||
let cmd = "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
|
||||
let
|
||||
# Place build output in 'build' folder, even if name includes a longer path.
|
||||
outName = os.lastPathPart(name)
|
||||
cmd = "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir & name & ".nim"
|
||||
|
||||
exec(cmd)
|
||||
|
||||
proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
|
||||
@ -24,6 +29,9 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
|
||||
task codex, "build codex binary":
|
||||
buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
|
||||
|
||||
task toolsCirdl, "build tools/cirdl binary":
|
||||
buildBinary "tools/cirdl/cirdl"
|
||||
|
||||
task testCodex, "Build & run Codex tests":
|
||||
test "testCodex", params = "-d:codex_enable_proof_failures=true"
|
||||
|
||||
@ -40,10 +48,15 @@ task build, "build codex binary":
|
||||
task test, "Run tests":
|
||||
testCodexTask()
|
||||
|
||||
task testTools, "Run Tools tests":
|
||||
toolsCirdlTask()
|
||||
test "testTools"
|
||||
|
||||
task testAll, "Run all tests (except for Taiko L2 tests)":
|
||||
testCodexTask()
|
||||
testContractsTask()
|
||||
testIntegrationTask()
|
||||
testToolsTask()
|
||||
|
||||
task testTaiko, "Run Taiko L2 tests":
|
||||
codexTask()
|
||||
|
||||
@ -67,6 +67,9 @@ when isMainModule:
|
||||
# permissions are insecure.
|
||||
quit QuitFailure
|
||||
|
||||
if config.prover() and not(checkAndCreateDataDir((config.circuitDir).string)):
|
||||
quit QuitFailure
|
||||
|
||||
trace "Data dir initialized", dir = $config.dataDir
|
||||
|
||||
if not(checkAndCreateDataDir((config.dataDir / "repo"))):
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import ./engine/discovery
|
||||
import ./engine/advertiser
|
||||
import ./engine/engine
|
||||
import ./engine/payments
|
||||
|
||||
export discovery, engine, payments
|
||||
export discovery, advertiser, engine, payments
|
||||
|
||||
177
codex/blockexchange/engine/advertiser.nim
Normal file
177
codex/blockexchange/engine/advertiser.nim
Normal file
@ -0,0 +1,177 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2022 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/libp2p/cid
|
||||
import pkg/libp2p/multicodec
|
||||
import pkg/metrics
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import ../protobuf/presence
|
||||
import ../peers
|
||||
|
||||
import ../../utils
|
||||
import ../../discovery
|
||||
import ../../stores/blockstore
|
||||
import ../../logutils
|
||||
import ../../manifest
|
||||
|
||||
logScope:
|
||||
topics = "codex discoveryengine advertiser"
|
||||
|
||||
declareGauge(codexInflightAdvertise, "inflight advertise requests")
|
||||
|
||||
const
|
||||
DefaultConcurrentAdvertRequests = 10
|
||||
DefaultAdvertiseLoopSleep = 30.minutes
|
||||
|
||||
type
|
||||
Advertiser* = ref object of RootObj
|
||||
localStore*: BlockStore # Local block store for this instance
|
||||
discovery*: Discovery # Discovery interface
|
||||
|
||||
advertiserRunning*: bool # Indicates if discovery is running
|
||||
concurrentAdvReqs: int # Concurrent advertise requests
|
||||
|
||||
advertiseLocalStoreLoop*: Future[void] # Advertise loop task handle
|
||||
advertiseQueue*: AsyncQueue[Cid] # Advertise queue
|
||||
advertiseTasks*: seq[Future[void]] # Advertise tasks
|
||||
|
||||
advertiseLocalStoreLoopSleep: Duration # Advertise loop sleep
|
||||
inFlightAdvReqs*: Table[Cid, Future[void]] # Inflight advertise requests
|
||||
|
||||
proc addCidToQueue(b: Advertiser, cid: Cid) {.async.} =
|
||||
if cid notin b.advertiseQueue:
|
||||
await b.advertiseQueue.put(cid)
|
||||
trace "Advertising", cid
|
||||
|
||||
proc advertiseBlock(b: Advertiser, cid: Cid) {.async.} =
|
||||
without isM =? cid.isManifest, err:
|
||||
warn "Unable to determine if cid is manifest"
|
||||
return
|
||||
|
||||
if isM:
|
||||
without blk =? await b.localStore.getBlock(cid), err:
|
||||
error "Error retrieving manifest block", cid, err = err.msg
|
||||
return
|
||||
|
||||
without manifest =? Manifest.decode(blk), err:
|
||||
error "Unable to decode as manifest", err = err.msg
|
||||
return
|
||||
|
||||
# announce manifest cid and tree cid
|
||||
await b.addCidToQueue(cid)
|
||||
await b.addCidToQueue(manifest.treeCid)
|
||||
|
||||
proc advertiseLocalStoreLoop(b: Advertiser) {.async.} =
|
||||
while b.advertiserRunning:
|
||||
if cids =? await b.localStore.listBlocks(blockType = BlockType.Manifest):
|
||||
trace "Advertiser begins iterating blocks..."
|
||||
for c in cids:
|
||||
if cid =? await c:
|
||||
await b.advertiseBlock(cid)
|
||||
trace "Advertiser iterating blocks finished."
|
||||
|
||||
await sleepAsync(b.advertiseLocalStoreLoopSleep)
|
||||
|
||||
info "Exiting advertise task loop"
|
||||
|
||||
proc processQueueLoop(b: Advertiser) {.async.} =
|
||||
while b.advertiserRunning:
|
||||
try:
|
||||
let
|
||||
cid = await b.advertiseQueue.get()
|
||||
|
||||
if cid in b.inFlightAdvReqs:
|
||||
continue
|
||||
|
||||
try:
|
||||
let
|
||||
request = b.discovery.provide(cid)
|
||||
|
||||
b.inFlightAdvReqs[cid] = request
|
||||
codexInflightAdvertise.set(b.inFlightAdvReqs.len.int64)
|
||||
await request
|
||||
|
||||
finally:
|
||||
b.inFlightAdvReqs.del(cid)
|
||||
codexInflightAdvertise.set(b.inFlightAdvReqs.len.int64)
|
||||
except CancelledError:
|
||||
trace "Advertise task cancelled"
|
||||
return
|
||||
except CatchableError as exc:
|
||||
warn "Exception in advertise task runner", exc = exc.msg
|
||||
|
||||
info "Exiting advertise task runner"
|
||||
|
||||
proc start*(b: Advertiser) {.async.} =
|
||||
## Start the advertiser
|
||||
##
|
||||
|
||||
trace "Advertiser start"
|
||||
|
||||
proc onBlock(cid: Cid) {.async.} =
|
||||
await b.advertiseBlock(cid)
|
||||
|
||||
doAssert(b.localStore.onBlockStored.isNone())
|
||||
b.localStore.onBlockStored = onBlock.some
|
||||
|
||||
if b.advertiserRunning:
|
||||
warn "Starting advertiser twice"
|
||||
return
|
||||
|
||||
b.advertiserRunning = true
|
||||
for i in 0..<b.concurrentAdvReqs:
|
||||
b.advertiseTasks.add(processQueueLoop(b))
|
||||
|
||||
b.advertiseLocalStoreLoop = advertiseLocalStoreLoop(b)
|
||||
|
||||
proc stop*(b: Advertiser) {.async.} =
|
||||
## Stop the advertiser
|
||||
##
|
||||
|
||||
trace "Advertiser stop"
|
||||
if not b.advertiserRunning:
|
||||
warn "Stopping advertiser without starting it"
|
||||
return
|
||||
|
||||
b.advertiserRunning = false
|
||||
# Stop incoming tasks from callback and localStore loop
|
||||
b.localStore.onBlockStored = CidCallback.none
|
||||
if not b.advertiseLocalStoreLoop.isNil and not b.advertiseLocalStoreLoop.finished:
|
||||
trace "Awaiting advertise loop to stop"
|
||||
await b.advertiseLocalStoreLoop.cancelAndWait()
|
||||
trace "Advertise loop stopped"
|
||||
|
||||
# Clear up remaining tasks
|
||||
for task in b.advertiseTasks:
|
||||
if not task.finished:
|
||||
trace "Awaiting advertise task to stop"
|
||||
await task.cancelAndWait()
|
||||
trace "Advertise task stopped"
|
||||
|
||||
trace "Advertiser stopped"
|
||||
|
||||
proc new*(
|
||||
T: type Advertiser,
|
||||
localStore: BlockStore,
|
||||
discovery: Discovery,
|
||||
concurrentAdvReqs = DefaultConcurrentAdvertRequests,
|
||||
advertiseLocalStoreLoopSleep = DefaultAdvertiseLoopSleep
|
||||
): Advertiser =
|
||||
## Create a advertiser instance
|
||||
##
|
||||
Advertiser(
|
||||
localStore: localStore,
|
||||
discovery: discovery,
|
||||
concurrentAdvReqs: concurrentAdvReqs,
|
||||
advertiseQueue: newAsyncQueue[Cid](concurrentAdvReqs),
|
||||
inFlightAdvReqs: initTable[Cid, Future[void]](),
|
||||
advertiseLocalStoreLoopSleep: advertiseLocalStoreLoopSleep)
|
||||
@ -35,11 +35,9 @@ declareGauge(codexInflightDiscovery, "inflight discovery requests")
|
||||
|
||||
const
|
||||
DefaultConcurrentDiscRequests = 10
|
||||
DefaultConcurrentAdvertRequests = 10
|
||||
DefaultDiscoveryTimeout = 1.minutes
|
||||
DefaultMinPeersPerBlock = 3
|
||||
DefaultDiscoveryLoopSleep = 3.seconds
|
||||
DefaultAdvertiseLoopSleep = 30.minutes
|
||||
|
||||
type
|
||||
DiscoveryEngine* = ref object of RootObj
|
||||
@ -49,20 +47,13 @@ type
|
||||
discovery*: Discovery # Discovery interface
|
||||
pendingBlocks*: PendingBlocksManager # Blocks we're awaiting to be resolved
|
||||
discEngineRunning*: bool # Indicates if discovery is running
|
||||
concurrentAdvReqs: int # Concurrent advertise requests
|
||||
concurrentDiscReqs: int # Concurrent discovery requests
|
||||
advertiseLoop*: Future[void] # Advertise loop task handle
|
||||
advertiseQueue*: AsyncQueue[Cid] # Advertise queue
|
||||
advertiseTasks*: seq[Future[void]] # Advertise tasks
|
||||
discoveryLoop*: Future[void] # Discovery loop task handle
|
||||
discoveryQueue*: AsyncQueue[Cid] # Discovery queue
|
||||
discoveryTasks*: seq[Future[void]] # Discovery tasks
|
||||
minPeersPerBlock*: int # Max number of peers with block
|
||||
discoveryLoopSleep: Duration # Discovery loop sleep
|
||||
advertiseLoopSleep: Duration # Advertise loop sleep
|
||||
inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]] # Inflight discovery requests
|
||||
inFlightAdvReqs*: Table[Cid, Future[void]] # Inflight advertise requests
|
||||
advertiseType*: BlockType # Advertice blocks, manifests or both
|
||||
|
||||
proc discoveryQueueLoop(b: DiscoveryEngine) {.async.} =
|
||||
while b.discEngineRunning:
|
||||
@ -81,69 +72,6 @@ proc discoveryQueueLoop(b: DiscoveryEngine) {.async.} =
|
||||
|
||||
await sleepAsync(b.discoveryLoopSleep)
|
||||
|
||||
proc advertiseBlock(b: DiscoveryEngine, cid: Cid) {.async.} =
|
||||
without isM =? cid.isManifest, err:
|
||||
warn "Unable to determine if cid is manifest"
|
||||
return
|
||||
|
||||
if isM:
|
||||
without blk =? await b.localStore.getBlock(cid), err:
|
||||
error "Error retrieving manifest block", cid, err = err.msg
|
||||
return
|
||||
|
||||
without manifest =? Manifest.decode(blk), err:
|
||||
error "Unable to decode as manifest", err = err.msg
|
||||
return
|
||||
|
||||
# announce manifest cid and tree cid
|
||||
await b.advertiseQueue.put(cid)
|
||||
await b.advertiseQueue.put(manifest.treeCid)
|
||||
|
||||
proc advertiseQueueLoop(b: DiscoveryEngine) {.async.} =
|
||||
while b.discEngineRunning:
|
||||
if cids =? await b.localStore.listBlocks(blockType = b.advertiseType):
|
||||
trace "Begin iterating blocks..."
|
||||
for c in cids:
|
||||
if cid =? await c:
|
||||
b.advertiseBlock(cid)
|
||||
await sleepAsync(100.millis)
|
||||
trace "Iterating blocks finished."
|
||||
|
||||
await sleepAsync(b.advertiseLoopSleep)
|
||||
|
||||
info "Exiting advertise task loop"
|
||||
|
||||
proc advertiseTaskLoop(b: DiscoveryEngine) {.async.} =
|
||||
## Run advertise tasks
|
||||
##
|
||||
|
||||
while b.discEngineRunning:
|
||||
try:
|
||||
let
|
||||
cid = await b.advertiseQueue.get()
|
||||
|
||||
if cid in b.inFlightAdvReqs:
|
||||
continue
|
||||
|
||||
try:
|
||||
let
|
||||
request = b.discovery.provide(cid)
|
||||
|
||||
b.inFlightAdvReqs[cid] = request
|
||||
codexInflightDiscovery.set(b.inFlightAdvReqs.len.int64)
|
||||
await request
|
||||
|
||||
finally:
|
||||
b.inFlightAdvReqs.del(cid)
|
||||
codexInflightDiscovery.set(b.inFlightAdvReqs.len.int64)
|
||||
except CancelledError:
|
||||
trace "Advertise task cancelled"
|
||||
return
|
||||
except CatchableError as exc:
|
||||
warn "Exception in advertise task runner", exc = exc.msg
|
||||
|
||||
info "Exiting advertise task runner"
|
||||
|
||||
proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} =
|
||||
## Run discovery tasks
|
||||
##
|
||||
@ -168,7 +96,7 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} =
|
||||
.wait(DefaultDiscoveryTimeout)
|
||||
|
||||
b.inFlightDiscReqs[cid] = request
|
||||
codexInflightDiscovery.set(b.inFlightAdvReqs.len.int64)
|
||||
codexInflightDiscovery.set(b.inFlightDiscReqs.len.int64)
|
||||
let
|
||||
peers = await request
|
||||
|
||||
@ -182,7 +110,7 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} =
|
||||
|
||||
finally:
|
||||
b.inFlightDiscReqs.del(cid)
|
||||
codexInflightDiscovery.set(b.inFlightAdvReqs.len.int64)
|
||||
codexInflightDiscovery.set(b.inFlightDiscReqs.len.int64)
|
||||
except CancelledError:
|
||||
trace "Discovery task cancelled"
|
||||
return
|
||||
@ -199,14 +127,6 @@ proc queueFindBlocksReq*(b: DiscoveryEngine, cids: seq[Cid]) {.inline.} =
|
||||
except CatchableError as exc:
|
||||
warn "Exception queueing discovery request", exc = exc.msg
|
||||
|
||||
proc queueProvideBlocksReq*(b: DiscoveryEngine, cids: seq[Cid]) {.inline.} =
|
||||
for cid in cids:
|
||||
if cid notin b.advertiseQueue:
|
||||
try:
|
||||
b.advertiseQueue.putNoWait(cid)
|
||||
except CatchableError as exc:
|
||||
warn "Exception queueing discovery request", exc = exc.msg
|
||||
|
||||
proc start*(b: DiscoveryEngine) {.async.} =
|
||||
## Start the discengine task
|
||||
##
|
||||
@ -218,13 +138,9 @@ proc start*(b: DiscoveryEngine) {.async.} =
|
||||
return
|
||||
|
||||
b.discEngineRunning = true
|
||||
for i in 0..<b.concurrentAdvReqs:
|
||||
b.advertiseTasks.add(advertiseTaskLoop(b))
|
||||
|
||||
for i in 0..<b.concurrentDiscReqs:
|
||||
b.discoveryTasks.add(discoveryTaskLoop(b))
|
||||
|
||||
b.advertiseLoop = advertiseQueueLoop(b)
|
||||
b.discoveryLoop = discoveryQueueLoop(b)
|
||||
|
||||
proc stop*(b: DiscoveryEngine) {.async.} =
|
||||
@ -237,23 +153,12 @@ proc stop*(b: DiscoveryEngine) {.async.} =
|
||||
return
|
||||
|
||||
b.discEngineRunning = false
|
||||
for task in b.advertiseTasks:
|
||||
if not task.finished:
|
||||
trace "Awaiting advertise task to stop"
|
||||
await task.cancelAndWait()
|
||||
trace "Advertise task stopped"
|
||||
|
||||
for task in b.discoveryTasks:
|
||||
if not task.finished:
|
||||
trace "Awaiting discovery task to stop"
|
||||
await task.cancelAndWait()
|
||||
trace "Discovery task stopped"
|
||||
|
||||
if not b.advertiseLoop.isNil and not b.advertiseLoop.finished:
|
||||
trace "Awaiting advertise loop to stop"
|
||||
await b.advertiseLoop.cancelAndWait()
|
||||
trace "Advertise loop stopped"
|
||||
|
||||
if not b.discoveryLoop.isNil and not b.discoveryLoop.finished:
|
||||
trace "Awaiting discovery loop to stop"
|
||||
await b.discoveryLoop.cancelAndWait()
|
||||
@ -268,12 +173,9 @@ proc new*(
|
||||
network: BlockExcNetwork,
|
||||
discovery: Discovery,
|
||||
pendingBlocks: PendingBlocksManager,
|
||||
concurrentAdvReqs = DefaultConcurrentAdvertRequests,
|
||||
concurrentDiscReqs = DefaultConcurrentDiscRequests,
|
||||
discoveryLoopSleep = DefaultDiscoveryLoopSleep,
|
||||
advertiseLoopSleep = DefaultAdvertiseLoopSleep,
|
||||
minPeersPerBlock = DefaultMinPeersPerBlock,
|
||||
advertiseType = BlockType.Manifest
|
||||
minPeersPerBlock = DefaultMinPeersPerBlock
|
||||
): DiscoveryEngine =
|
||||
## Create a discovery engine instance for advertising services
|
||||
##
|
||||
@ -283,13 +185,8 @@ proc new*(
|
||||
network: network,
|
||||
discovery: discovery,
|
||||
pendingBlocks: pendingBlocks,
|
||||
concurrentAdvReqs: concurrentAdvReqs,
|
||||
concurrentDiscReqs: concurrentDiscReqs,
|
||||
advertiseQueue: newAsyncQueue[Cid](concurrentAdvReqs),
|
||||
discoveryQueue: newAsyncQueue[Cid](concurrentDiscReqs),
|
||||
inFlightDiscReqs: initTable[Cid, Future[seq[SignedPeerRecord]]](),
|
||||
inFlightAdvReqs: initTable[Cid, Future[void]](),
|
||||
discoveryLoopSleep: discoveryLoopSleep,
|
||||
advertiseLoopSleep: advertiseLoopSleep,
|
||||
minPeersPerBlock: minPeersPerBlock,
|
||||
advertiseType: advertiseType)
|
||||
minPeersPerBlock: minPeersPerBlock)
|
||||
|
||||
@ -34,6 +34,7 @@ import ../peers
|
||||
|
||||
import ./payments
|
||||
import ./discovery
|
||||
import ./advertiser
|
||||
import ./pendingblocks
|
||||
|
||||
export peers, pendingblocks, payments, discovery
|
||||
@ -77,6 +78,7 @@ type
|
||||
pricing*: ?Pricing # Optional bandwidth pricing
|
||||
blockFetchTimeout*: Duration # Timeout for fetching blocks over the network
|
||||
discovery*: DiscoveryEngine
|
||||
advertiser*: Advertiser
|
||||
|
||||
Pricing* = object
|
||||
address*: EthAddress
|
||||
@ -93,6 +95,7 @@ proc start*(b: BlockExcEngine) {.async.} =
|
||||
##
|
||||
|
||||
await b.discovery.start()
|
||||
await b.advertiser.start()
|
||||
|
||||
trace "Blockexc starting with concurrent tasks", tasks = b.concurrentTasks
|
||||
if b.blockexcRunning:
|
||||
@ -108,6 +111,7 @@ proc stop*(b: BlockExcEngine) {.async.} =
|
||||
##
|
||||
|
||||
await b.discovery.stop()
|
||||
await b.advertiser.stop()
|
||||
|
||||
trace "NetworkStore stop"
|
||||
if not b.blockexcRunning:
|
||||
@ -284,27 +288,11 @@ proc cancelBlocks(b: BlockExcEngine, addrs: seq[BlockAddress]) {.async.} =
|
||||
if failed.len > 0:
|
||||
warn "Failed to send block request cancellations to peers", peers = failed.len
|
||||
|
||||
proc getAnnouceCids(blocksDelivery: seq[BlockDelivery]): seq[Cid] =
|
||||
var cids = initHashSet[Cid]()
|
||||
for bd in blocksDelivery:
|
||||
if bd.address.leaf:
|
||||
cids.incl(bd.address.treeCid)
|
||||
else:
|
||||
without isM =? bd.address.cid.isManifest, err:
|
||||
warn "Unable to determine if cid is manifest"
|
||||
continue
|
||||
if isM:
|
||||
cids.incl(bd.address.cid)
|
||||
return cids.toSeq
|
||||
|
||||
proc resolveBlocks*(b: BlockExcEngine, blocksDelivery: seq[BlockDelivery]) {.async.} =
|
||||
b.pendingBlocks.resolve(blocksDelivery)
|
||||
await b.scheduleTasks(blocksDelivery)
|
||||
let announceCids = getAnnouceCids(blocksDelivery)
|
||||
await b.cancelBlocks(blocksDelivery.mapIt(it.address))
|
||||
|
||||
b.discovery.queueProvideBlocksReq(announceCids)
|
||||
|
||||
proc resolveBlocks*(b: BlockExcEngine, blocks: seq[Block]) {.async.} =
|
||||
await b.resolveBlocks(
|
||||
blocks.mapIt(
|
||||
@ -596,6 +584,7 @@ proc new*(
|
||||
wallet: WalletRef,
|
||||
network: BlockExcNetwork,
|
||||
discovery: DiscoveryEngine,
|
||||
advertiser: Advertiser,
|
||||
peerStore: PeerCtxStore,
|
||||
pendingBlocks: PendingBlocksManager,
|
||||
concurrentTasks = DefaultConcurrentTasks,
|
||||
@ -616,6 +605,7 @@ proc new*(
|
||||
concurrentTasks: concurrentTasks,
|
||||
taskQueue: newAsyncHeapQueue[BlockExcPeerCtx](DefaultTaskQueueSize),
|
||||
discovery: discovery,
|
||||
advertiser: advertiser,
|
||||
blockFetchTimeout: blockFetchTimeout)
|
||||
|
||||
proc peerEventHandler(peerId: PeerId, event: PeerEvent) {.async.} =
|
||||
|
||||
@ -93,18 +93,20 @@ proc send*(b: BlockExcNetwork, id: PeerId, msg: pb.Message) {.async.} =
|
||||
## Send message to peer
|
||||
##
|
||||
|
||||
b.peers.withValue(id, peer):
|
||||
try:
|
||||
await b.inflightSema.acquire()
|
||||
await peer[].send(msg)
|
||||
except CancelledError as error:
|
||||
raise error
|
||||
except CatchableError as err:
|
||||
error "Error sending message", peer = id, msg = err.msg
|
||||
finally:
|
||||
b.inflightSema.release()
|
||||
do:
|
||||
if not (id in b.peers):
|
||||
trace "Unable to send, peer not found", peerId = id
|
||||
return
|
||||
|
||||
let peer = b.peers[id]
|
||||
try:
|
||||
await b.inflightSema.acquire()
|
||||
await peer.send(msg)
|
||||
except CancelledError as error:
|
||||
raise error
|
||||
except CatchableError as err:
|
||||
error "Error sending message", peer = id, msg = err.msg
|
||||
finally:
|
||||
b.inflightSema.release()
|
||||
|
||||
proc handleWantList(
|
||||
b: BlockExcNetwork,
|
||||
|
||||
@ -31,7 +31,7 @@ import ./codextypes
|
||||
export errors, logutils, units, codextypes
|
||||
|
||||
type
|
||||
Block* = object of RootObj
|
||||
Block* = ref object of RootObj
|
||||
cid*: Cid
|
||||
data*: seq[byte]
|
||||
|
||||
|
||||
@ -110,7 +110,7 @@ proc bootstrapInteractions(
|
||||
quit QuitFailure
|
||||
|
||||
let marketplace = Marketplace.new(marketplaceAddress, signer)
|
||||
let market = OnChainMarket.new(marketplace)
|
||||
let market = OnChainMarket.new(marketplace, config.rewardRecipient)
|
||||
let clock = OnChainClock.new(provider)
|
||||
|
||||
var client: ?ClientInteractions
|
||||
@ -122,25 +122,30 @@ proc bootstrapInteractions(
|
||||
else:
|
||||
s.codexNode.clock = SystemClock()
|
||||
|
||||
if config.persistence:
|
||||
# This is used for simulation purposes. Normal nodes won't be compiled with this flag
|
||||
# and hence the proof failure will always be 0.
|
||||
when codex_enable_proof_failures:
|
||||
let proofFailures = config.simulateProofFailures
|
||||
if proofFailures > 0:
|
||||
warn "Enabling proof failure simulation!"
|
||||
else:
|
||||
let proofFailures = 0
|
||||
if config.simulateProofFailures > 0:
|
||||
warn "Proof failure simulation is not enabled for this build! Configuration ignored"
|
||||
# This is used for simulation purposes. Normal nodes won't be compiled with this flag
|
||||
# and hence the proof failure will always be 0.
|
||||
when codex_enable_proof_failures:
|
||||
let proofFailures = config.simulateProofFailures
|
||||
if proofFailures > 0:
|
||||
warn "Enabling proof failure simulation!"
|
||||
else:
|
||||
let proofFailures = 0
|
||||
if config.simulateProofFailures > 0:
|
||||
warn "Proof failure simulation is not enabled for this build! Configuration ignored"
|
||||
|
||||
let purchasing = Purchasing.new(market, clock)
|
||||
let sales = Sales.new(market, clock, repo, proofFailures)
|
||||
client = some ClientInteractions.new(clock, purchasing)
|
||||
host = some HostInteractions.new(clock, sales)
|
||||
let purchasing = Purchasing.new(market, clock)
|
||||
let sales = Sales.new(market, clock, repo, proofFailures)
|
||||
client = some ClientInteractions.new(clock, purchasing)
|
||||
host = some HostInteractions.new(clock, sales)
|
||||
|
||||
if config.validator:
|
||||
let validation = Validation.new(clock, market, config.validatorMaxSlots)
|
||||
without validationConfig =? ValidationConfig.init(
|
||||
config.validatorMaxSlots,
|
||||
config.validatorGroups,
|
||||
config.validatorGroupIndex), err:
|
||||
error "Invalid validation parameters", err = err.msg
|
||||
quit QuitFailure
|
||||
let validation = Validation.new(clock, market, validationConfig)
|
||||
validator = some ValidatorInteractions.new(clock, validation)
|
||||
|
||||
s.codexNode.contracts = (client, host, validator)
|
||||
@ -258,7 +263,7 @@ proc new*(
|
||||
repoDs = repoData,
|
||||
metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace)
|
||||
.expect("Should create metadata store!"),
|
||||
quotaMaxBytes = config.storageQuota.uint,
|
||||
quotaMaxBytes = config.storageQuota,
|
||||
blockTtl = config.blockTtl)
|
||||
|
||||
maintenance = BlockMaintainer.new(
|
||||
@ -268,36 +273,13 @@ proc new*(
|
||||
|
||||
peerStore = PeerCtxStore.new()
|
||||
pendingBlocks = PendingBlocksManager.new()
|
||||
advertiser = Advertiser.new(repoStore, discovery)
|
||||
blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks)
|
||||
engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, peerStore, pendingBlocks)
|
||||
engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks)
|
||||
store = NetworkStore.new(engine, repoStore)
|
||||
prover = if config.prover:
|
||||
if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) and
|
||||
endsWith($config.circomR1cs, ".r1cs"):
|
||||
error "Circom R1CS file not accessible"
|
||||
raise (ref Defect)(
|
||||
msg: "r1cs file not readable, doesn't exist or wrong extension (.r1cs)")
|
||||
|
||||
if not fileAccessible($config.circomWasm, {AccessFlags.Read}) and
|
||||
endsWith($config.circomWasm, ".wasm"):
|
||||
error "Circom wasm file not accessible"
|
||||
raise (ref Defect)(
|
||||
msg: "wasm file not readable, doesn't exist or wrong extension (.wasm)")
|
||||
|
||||
let zkey = if not config.circomNoZkey:
|
||||
if not fileAccessible($config.circomZkey, {AccessFlags.Read}) and
|
||||
endsWith($config.circomZkey, ".zkey"):
|
||||
error "Circom zkey file not accessible"
|
||||
raise (ref Defect)(
|
||||
msg: "zkey file not readable, doesn't exist or wrong extension (.zkey)")
|
||||
|
||||
$config.circomZkey
|
||||
else: ""
|
||||
|
||||
some Prover.new(
|
||||
store,
|
||||
CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey),
|
||||
config.numProofSamples)
|
||||
let backend = config.initializeBackend().expect("Unable to create prover backend.")
|
||||
some Prover.new(store, backend, config.numProofSamples)
|
||||
else:
|
||||
none Prover
|
||||
|
||||
|
||||
@ -37,8 +37,10 @@ import ./logutils
|
||||
import ./stores
|
||||
import ./units
|
||||
import ./utils
|
||||
from ./validationconfig import MaxSlots, ValidationGroups
|
||||
|
||||
export units, net, codextypes, logutils
|
||||
export ValidationGroups, MaxSlots
|
||||
|
||||
export
|
||||
DefaultQuotaBytes,
|
||||
@ -62,6 +64,7 @@ const
|
||||
codex_enable_log_counter* {.booldefine.} = false
|
||||
|
||||
DefaultDataDir* = defaultDataDir()
|
||||
DefaultCircuitDir* = defaultDataDir() / "circuits"
|
||||
|
||||
type
|
||||
StartUpCmd* {.pure.} = enum
|
||||
@ -98,7 +101,8 @@ type
|
||||
|
||||
logFormat* {.
|
||||
hidden
|
||||
desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
|
||||
desc: "Specifies what kind of logs should be written to stdout (auto, " &
|
||||
"colors, nocolors, json)"
|
||||
defaultValueDesc: "auto"
|
||||
defaultValue: LogKind.Auto
|
||||
name: "log-format" }: LogKind
|
||||
@ -163,7 +167,8 @@ type
|
||||
name: "net-privkey" }: string
|
||||
|
||||
bootstrapNodes* {.
|
||||
desc: "Specifies one or more bootstrap nodes to use when connecting to the network"
|
||||
desc: "Specifies one or more bootstrap nodes to use when " &
|
||||
"connecting to the network"
|
||||
abbr: "b"
|
||||
name: "bootstrap-node" }: seq[SignedPeerRecord]
|
||||
|
||||
@ -191,7 +196,8 @@ type
|
||||
abbr: "p" }: Port
|
||||
|
||||
apiCorsAllowedOrigin* {.
|
||||
desc: "The REST Api CORS allowed origin for downloading data. '*' will allow all origins, '' will allow none.",
|
||||
desc: "The REST Api CORS allowed origin for downloading data. " &
|
||||
"'*' will allow all origins, '' will allow none.",
|
||||
defaultValue: string.none
|
||||
defaultValueDesc: "Disallow all cross origin requests to download data"
|
||||
name: "api-cors-origin" }: Option[string]
|
||||
@ -217,7 +223,9 @@ type
|
||||
abbr: "t" }: Duration
|
||||
|
||||
blockMaintenanceInterval* {.
|
||||
desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup"
|
||||
desc: "Time interval in seconds - determines frequency of block " &
|
||||
"maintenance cycle: how often blocks are checked " &
|
||||
"for expiration and cleanup"
|
||||
defaultValue: DefaultBlockMaintenanceInterval
|
||||
defaultValueDesc: $DefaultBlockMaintenanceInterval
|
||||
name: "block-mi" }: Duration
|
||||
@ -229,7 +237,8 @@ type
|
||||
name: "block-mn" }: int
|
||||
|
||||
cacheSize* {.
|
||||
desc: "The size of the block cache, 0 disables the cache - might help on slow hardrives"
|
||||
desc: "The size of the block cache, 0 disables the cache - " &
|
||||
"might help on slow hardrives"
|
||||
defaultValue: 0
|
||||
defaultValueDesc: "0"
|
||||
name: "cache-size"
|
||||
@ -289,32 +298,70 @@ type
|
||||
|
||||
validatorMaxSlots* {.
|
||||
desc: "Maximum number of slots that the validator monitors"
|
||||
longDesc: "If set to 0, the validator will not limit " &
|
||||
"the maximum number of slots it monitors"
|
||||
defaultValue: 1000
|
||||
name: "validator-max-slots"
|
||||
.}: int
|
||||
.}: MaxSlots
|
||||
|
||||
validatorGroups* {.
|
||||
desc: "Slot validation groups"
|
||||
longDesc: "A number indicating total number of groups into " &
|
||||
"which the whole slot id space will be divided. " &
|
||||
"The value must be in the range [2, 65535]. " &
|
||||
"If not provided, the validator will observe " &
|
||||
"the whole slot id space and the value of " &
|
||||
"the --validator-group-index parameter will be ignored. " &
|
||||
"Powers of twos are advised for even distribution"
|
||||
defaultValue: ValidationGroups.none
|
||||
name: "validator-groups"
|
||||
.}: Option[ValidationGroups]
|
||||
|
||||
validatorGroupIndex* {.
|
||||
desc: "Slot validation group index"
|
||||
longDesc: "The value provided must be in the range " &
|
||||
"[0, validatorGroups). Ignored when --validator-groups " &
|
||||
"is not provided. Only slot ids satisfying condition " &
|
||||
"[(slotId mod validationGroups) == groupIndex] will be " &
|
||||
"observed by the validator"
|
||||
defaultValue: 0
|
||||
name: "validator-group-index"
|
||||
.}: uint16
|
||||
|
||||
rewardRecipient* {.
|
||||
desc: "Address to send payouts to (eg rewards and refunds)"
|
||||
name: "reward-recipient"
|
||||
.}: Option[EthAddress]
|
||||
|
||||
case persistenceCmd* {.
|
||||
defaultValue: noCmd
|
||||
command }: PersistenceCmd
|
||||
|
||||
of PersistenceCmd.prover:
|
||||
circuitDir* {.
|
||||
desc: "Directory where Codex will store proof circuit data"
|
||||
defaultValue: DefaultCircuitDir
|
||||
defaultValueDesc: $DefaultCircuitDir
|
||||
abbr: "cd"
|
||||
name: "circuit-dir" }: OutDir
|
||||
|
||||
circomR1cs* {.
|
||||
desc: "The r1cs file for the storage circuit"
|
||||
defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs"
|
||||
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs"
|
||||
defaultValue: $DefaultCircuitDir / "proof_main.r1cs"
|
||||
defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs"
|
||||
name: "circom-r1cs"
|
||||
.}: InputFile
|
||||
|
||||
circomWasm* {.
|
||||
desc: "The wasm file for the storage circuit"
|
||||
defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm"
|
||||
defaultValue: $DefaultCircuitDir / "proof_main.wasm"
|
||||
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
|
||||
name: "circom-wasm"
|
||||
.}: InputFile
|
||||
|
||||
circomZkey* {.
|
||||
desc: "The zkey file for the storage circuit"
|
||||
defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey"
|
||||
defaultValue: $DefaultCircuitDir / "proof_main.zkey"
|
||||
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
|
||||
name: "circom-zkey"
|
||||
.}: InputFile
|
||||
@ -533,7 +580,10 @@ proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} =
|
||||
try:
|
||||
setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii))
|
||||
except ValueError:
|
||||
raise (ref ValueError)(msg: "Please specify one of: trace, debug, info, notice, warn, error or fatal")
|
||||
raise (ref ValueError)(
|
||||
msg: "Please specify one of: trace, debug, " &
|
||||
"info, notice, warn, error or fatal"
|
||||
)
|
||||
|
||||
if directives.len > 1:
|
||||
for topicName, settings in parseTopicDirectives(directives[1..^1]):
|
||||
|
||||
@ -18,6 +18,10 @@ type
|
||||
timeout*: UInt256 # mark proofs as missing before the timeout (in seconds)
|
||||
downtime*: uint8 # ignore this much recent blocks for proof requirements
|
||||
zkeyHash*: string # hash of the zkey file which is linked to the verifier
|
||||
# Ensures the pointer does not remain in downtime for many consecutive
|
||||
# periods. For each period increase, move the pointer `pointerProduct`
|
||||
# blocks. Should be a prime number to ensure there are no cycles.
|
||||
downtimeProduct*: uint8
|
||||
|
||||
|
||||
func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig =
|
||||
@ -25,7 +29,8 @@ func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig =
|
||||
period: tupl[0],
|
||||
timeout: tupl[1],
|
||||
downtime: tupl[2],
|
||||
zkeyHash: tupl[3]
|
||||
zkeyHash: tupl[3],
|
||||
downtimeProduct: tupl[4]
|
||||
)
|
||||
|
||||
func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig =
|
||||
|
||||
@ -19,6 +19,11 @@ const knownAddresses = {
|
||||
# Taiko Alpha-3 Testnet
|
||||
"167005": {
|
||||
"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")
|
||||
}.toTable,
|
||||
|
||||
# Codex Testnet - Oct 08 2024 08:02:50 (+0 UTC)
|
||||
"789987": {
|
||||
"Marketplace": Address.init("0xfE822Df439d987849a90B64a4C0e26a297DBD47F")
|
||||
}.toTable
|
||||
}.toTable
|
||||
|
||||
|
||||
@ -19,17 +19,24 @@ type
|
||||
OnChainMarket* = ref object of Market
|
||||
contract: Marketplace
|
||||
signer: Signer
|
||||
rewardRecipient: ?Address
|
||||
MarketSubscription = market.Subscription
|
||||
EventSubscription = ethers.Subscription
|
||||
OnChainMarketSubscription = ref object of MarketSubscription
|
||||
eventSubscription: EventSubscription
|
||||
|
||||
func new*(_: type OnChainMarket, contract: Marketplace): OnChainMarket =
|
||||
func new*(
|
||||
_: type OnChainMarket,
|
||||
contract: Marketplace,
|
||||
rewardRecipient = Address.none): OnChainMarket =
|
||||
|
||||
without signer =? contract.signer:
|
||||
raiseAssert("Marketplace contract should have a signer")
|
||||
|
||||
OnChainMarket(
|
||||
contract: contract,
|
||||
signer: signer,
|
||||
rewardRecipient: rewardRecipient
|
||||
)
|
||||
|
||||
proc raiseMarketError(message: string) {.raises: [MarketError].} =
|
||||
@ -163,7 +170,23 @@ method fillSlot(market: OnChainMarket,
|
||||
|
||||
method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
|
||||
convertEthersError:
|
||||
discard await market.contract.freeSlot(slotId).confirm(0)
|
||||
var freeSlot: Future[?TransactionResponse]
|
||||
if rewardRecipient =? market.rewardRecipient:
|
||||
# If --reward-recipient specified, use it as the reward recipient, and use
|
||||
# the SP's address as the collateral recipient
|
||||
let collateralRecipient = await market.getSigner()
|
||||
freeSlot = market.contract.freeSlot(
|
||||
slotId,
|
||||
rewardRecipient, # --reward-recipient
|
||||
collateralRecipient) # SP's address
|
||||
|
||||
else:
|
||||
# Otherwise, use the SP's address as both the reward and collateral
|
||||
# recipient (the contract will use msg.sender for both)
|
||||
freeSlot = market.contract.freeSlot(slotId)
|
||||
|
||||
discard await freeSlot.confirm(0)
|
||||
|
||||
|
||||
method withdrawFunds(market: OnChainMarket,
|
||||
requestId: RequestId) {.async.} =
|
||||
@ -224,6 +247,22 @@ method canProofBeMarkedAsMissing*(
|
||||
trace "Proof cannot be marked as missing", msg = e.msg
|
||||
return false
|
||||
|
||||
method reserveSlot*(
|
||||
market: OnChainMarket,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256) {.async.} =
|
||||
|
||||
convertEthersError:
|
||||
discard await market.contract.reserveSlot(requestId, slotIndex).confirm(0)
|
||||
|
||||
method canReserveSlot*(
|
||||
market: OnChainMarket,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256): Future[bool] {.async.} =
|
||||
|
||||
convertEthersError:
|
||||
return await market.contract.canReserveSlot(requestId, slotIndex)
|
||||
|
||||
method subscribeRequests*(market: OnChainMarket,
|
||||
callback: OnRequest):
|
||||
Future[MarketSubscription] {.async.} =
|
||||
@ -268,6 +307,17 @@ method subscribeSlotFreed*(market: OnChainMarket,
|
||||
let subscription = await market.contract.subscribe(SlotFreed, onEvent)
|
||||
return OnChainMarketSubscription(eventSubscription: subscription)
|
||||
|
||||
method subscribeSlotReservationsFull*(
|
||||
market: OnChainMarket,
|
||||
callback: OnSlotReservationsFull): Future[MarketSubscription] {.async.} =
|
||||
|
||||
proc onEvent(event: SlotReservationsFull) {.upraises:[].} =
|
||||
callback(event.requestId, event.slotIndex)
|
||||
|
||||
convertEthersError:
|
||||
let subscription = await market.contract.subscribe(SlotReservationsFull, onEvent)
|
||||
return OnChainMarketSubscription(eventSubscription: subscription)
|
||||
|
||||
method subscribeFulfillment(market: OnChainMarket,
|
||||
callback: OnFulfillment):
|
||||
Future[MarketSubscription] {.async.} =
|
||||
@ -347,9 +397,11 @@ method subscribeProofSubmission*(market: OnChainMarket,
|
||||
method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} =
|
||||
await subscription.eventSubscription.unsubscribe()
|
||||
|
||||
method queryPastStorageRequests*(market: OnChainMarket,
|
||||
blocksAgo: int):
|
||||
Future[seq[PastStorageRequest]] {.async.} =
|
||||
method queryPastEvents*[T: MarketplaceEvent](
|
||||
market: OnChainMarket,
|
||||
_: type T,
|
||||
blocksAgo: int): Future[seq[T]] {.async.} =
|
||||
|
||||
convertEthersError:
|
||||
let contract = market.contract
|
||||
let provider = contract.provider
|
||||
@ -357,13 +409,6 @@ method queryPastStorageRequests*(market: OnChainMarket,
|
||||
let head = await provider.getBlockNumber()
|
||||
let fromBlock = BlockTag.init(head - blocksAgo.abs.u256)
|
||||
|
||||
let events = await contract.queryFilter(StorageRequested,
|
||||
fromBlock,
|
||||
BlockTag.latest)
|
||||
return events.map(event =>
|
||||
PastStorageRequest(
|
||||
requestId: event.requestId,
|
||||
ask: event.ask,
|
||||
expiry: event.expiry
|
||||
)
|
||||
)
|
||||
return await contract.queryFilter(T,
|
||||
fromBlock,
|
||||
BlockTag.latest)
|
||||
|
||||
@ -16,25 +16,6 @@ export requests
|
||||
|
||||
type
|
||||
Marketplace* = ref object of Contract
|
||||
StorageRequested* = object of Event
|
||||
requestId*: RequestId
|
||||
ask*: StorageAsk
|
||||
expiry*: UInt256
|
||||
SlotFilled* = object of Event
|
||||
requestId* {.indexed.}: RequestId
|
||||
slotIndex*: UInt256
|
||||
SlotFreed* = object of Event
|
||||
requestId* {.indexed.}: RequestId
|
||||
slotIndex*: UInt256
|
||||
RequestFulfilled* = object of Event
|
||||
requestId* {.indexed.}: RequestId
|
||||
RequestCancelled* = object of Event
|
||||
requestId* {.indexed.}: RequestId
|
||||
RequestFailed* = object of Event
|
||||
requestId* {.indexed.}: RequestId
|
||||
ProofSubmitted* = object of Event
|
||||
id*: SlotId
|
||||
|
||||
|
||||
proc config*(marketplace: Marketplace): MarketplaceConfig {.contract, view.}
|
||||
proc token*(marketplace: Marketplace): Address {.contract, view.}
|
||||
@ -45,7 +26,9 @@ proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view
|
||||
proc requestStorage*(marketplace: Marketplace, request: StorageRequest): ?TransactionResponse {.contract.}
|
||||
proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): ?TransactionResponse {.contract.}
|
||||
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): ?TransactionResponse {.contract.}
|
||||
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): ?TransactionResponse {.contract.}
|
||||
proc freeSlot*(marketplace: Marketplace, id: SlotId): ?TransactionResponse {.contract.}
|
||||
proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): ?TransactionResponse {.contract.}
|
||||
proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view.}
|
||||
proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.}
|
||||
proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view.}
|
||||
@ -68,3 +51,6 @@ proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.}
|
||||
|
||||
proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): ?TransactionResponse {.contract.}
|
||||
proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): ?TransactionResponse {.contract.}
|
||||
|
||||
proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): ?TransactionResponse {.contract.}
|
||||
proc canReserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): bool {.contract, view.}
|
||||
|
||||
@ -163,12 +163,12 @@ func id*(request: StorageRequest): RequestId =
|
||||
let encoding = AbiEncoder.encode((request, ))
|
||||
RequestId(keccak256.digest(encoding).data)
|
||||
|
||||
func slotId*(requestId: RequestId, slot: UInt256): SlotId =
|
||||
let encoding = AbiEncoder.encode((requestId, slot))
|
||||
func slotId*(requestId: RequestId, slotIndex: UInt256): SlotId =
|
||||
let encoding = AbiEncoder.encode((requestId, slotIndex))
|
||||
SlotId(keccak256.digest(encoding).data)
|
||||
|
||||
func slotId*(request: StorageRequest, slot: UInt256): SlotId =
|
||||
slotId(request.id, slot)
|
||||
func slotId*(request: StorageRequest, slotIndex: UInt256): SlotId =
|
||||
slotId(request.id, slotIndex)
|
||||
|
||||
func id*(slot: Slot): SlotId =
|
||||
slotId(slot.request, slot.slotIndex)
|
||||
|
||||
@ -114,7 +114,14 @@ proc proxySpawnEncodeTask(
|
||||
args: EncodeTaskArgs,
|
||||
data: ref seq[seq[byte]]
|
||||
): Flowvar[EncodeTaskResult] =
|
||||
tp.spawn encodeTask(args, data[])
|
||||
# FIXME Uncomment the code below after addressing an issue:
|
||||
# https://github.com/codex-storage/nim-codex/issues/854
|
||||
|
||||
# tp.spawn encodeTask(args, data[])
|
||||
|
||||
let fv = EncodeTaskResult.newFlowVar
|
||||
fv.readyWith(encodeTask(args, data[]))
|
||||
return fv
|
||||
|
||||
proc proxySpawnDecodeTask(
|
||||
tp: Taskpool,
|
||||
@ -122,7 +129,14 @@ proc proxySpawnDecodeTask(
|
||||
data: ref seq[seq[byte]],
|
||||
parity: ref seq[seq[byte]]
|
||||
): Flowvar[DecodeTaskResult] =
|
||||
tp.spawn decodeTask(args, data[], parity[])
|
||||
# FIXME Uncomment the code below after addressing an issue:
|
||||
# https://github.com/codex-storage/nim-codex/issues/854
|
||||
|
||||
# tp.spawn decodeTask(args, data[], parity[])
|
||||
|
||||
let fv = DecodeTaskResult.newFlowVar
|
||||
fv.readyWith(decodeTask(args, data[], parity[]))
|
||||
return fv
|
||||
|
||||
proc awaitResult[T](signal: ThreadSignalPtr, handle: Flowvar[T]): Future[?!T] {.async.} =
|
||||
await wait(signal)
|
||||
|
||||
@ -27,6 +27,7 @@ import ../blocktype as bt
|
||||
import ../utils
|
||||
import ../utils/asynciter
|
||||
import ../indexingstrategy
|
||||
import ../errors
|
||||
|
||||
import pkg/stew/byteutils
|
||||
|
||||
@ -82,6 +83,13 @@ type
|
||||
blocksCount: Natural
|
||||
strategy: StrategyType
|
||||
|
||||
ErasureError* = object of CodexError
|
||||
InsufficientBlocksError* = object of ErasureError
|
||||
# Minimum size, in bytes, that the dataset must have had
|
||||
# for the encoding request to have succeeded with the parameters
|
||||
# provided.
|
||||
minSize*: NBytes
|
||||
|
||||
func indexToPos(steps, idx, step: int): int {.inline.} =
|
||||
## Convert an index to a position in the encoded
|
||||
## dataset
|
||||
@ -236,11 +244,13 @@ proc init*(
|
||||
ecK: Natural, ecM: Natural,
|
||||
strategy: StrategyType): ?!EncodingParams =
|
||||
if ecK > manifest.blocksCount:
|
||||
return failure(
|
||||
"Unable to encode manifest, not enough blocks, ecK = " &
|
||||
let exc = (ref InsufficientBlocksError)(
|
||||
msg: "Unable to encode manifest, not enough blocks, ecK = " &
|
||||
$ecK &
|
||||
", blocksCount = " &
|
||||
$manifest.blocksCount)
|
||||
$manifest.blocksCount,
|
||||
minSize: ecK.NBytes * manifest.blockSize)
|
||||
return failure(exc)
|
||||
|
||||
let
|
||||
rounded = roundUp(manifest.blocksCount, ecK)
|
||||
|
||||
@ -49,8 +49,8 @@ func getLinearIndicies(
|
||||
self.checkIteration(iteration)
|
||||
|
||||
let
|
||||
first = self.firstIndex + iteration * (self.step + 1)
|
||||
last = min(first + self.step, self.lastIndex)
|
||||
first = self.firstIndex + iteration * self.step
|
||||
last = min(first + self.step - 1, self.lastIndex)
|
||||
|
||||
getIter(first, last, 1)
|
||||
|
||||
@ -94,4 +94,4 @@ func init*(
|
||||
firstIndex: firstIndex,
|
||||
lastIndex: lastIndex,
|
||||
iterations: iterations,
|
||||
step: divUp((lastIndex - firstIndex), iterations))
|
||||
step: divUp((lastIndex - firstIndex + 1), iterations))
|
||||
|
||||
@ -29,7 +29,7 @@ import ../logutils
|
||||
# TODO: Manifest should be reworked to more concrete types,
|
||||
# perhaps using inheritance
|
||||
type
|
||||
Manifest* = object of RootObj
|
||||
Manifest* = ref object of RootObj
|
||||
treeCid {.serialize.}: Cid # Root of the merkle tree
|
||||
datasetSize {.serialize.}: NBytes # Total size of all blocks
|
||||
blockSize {.serialize.}: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed)
|
||||
@ -135,13 +135,6 @@ func isManifest*(mc: MultiCodec): ?!bool =
|
||||
# Various sizes and verification
|
||||
############################################################
|
||||
|
||||
func bytes*(self: Manifest, pad = true): NBytes =
|
||||
## Compute how many bytes corresponding StoreStream(Manifest, pad) will return
|
||||
if pad or self.protected:
|
||||
self.blocksCount.NBytes * self.blockSize
|
||||
else:
|
||||
self.datasetSize
|
||||
|
||||
func rounded*(self: Manifest): int =
|
||||
## Number of data blocks in *protected* manifest including padding at the end
|
||||
roundUp(self.originalBlocksCount, self.ecK)
|
||||
@ -238,7 +231,7 @@ func new*(
|
||||
treeCid: Cid,
|
||||
datasetSize: NBytes,
|
||||
ecK, ecM: int,
|
||||
strategy: StrategyType): Manifest =
|
||||
strategy = SteppedStrategy): Manifest =
|
||||
## Create an erasure protected dataset from an
|
||||
## unprotected one
|
||||
##
|
||||
@ -284,7 +277,7 @@ func new*(
|
||||
ecM: int,
|
||||
originalTreeCid: Cid,
|
||||
originalDatasetSize: NBytes,
|
||||
strategy: StrategyType): Manifest =
|
||||
strategy = SteppedStrategy): Manifest =
|
||||
|
||||
Manifest(
|
||||
treeCid: treeCid,
|
||||
@ -306,7 +299,7 @@ func new*(
|
||||
verifyRoot: Cid,
|
||||
slotRoots: openArray[Cid],
|
||||
cellSize = DefaultCellSize,
|
||||
strategy = SteppedStrategy): ?!Manifest =
|
||||
strategy = LinearStrategy): ?!Manifest =
|
||||
## Create a verifiable dataset from an
|
||||
## protected one
|
||||
##
|
||||
@ -329,8 +322,9 @@ func new*(
|
||||
protected: true,
|
||||
ecK: manifest.ecK,
|
||||
ecM: manifest.ecM,
|
||||
originalTreeCid: manifest.treeCid,
|
||||
originalTreeCid: manifest.originalTreeCid,
|
||||
originalDatasetSize: manifest.originalDatasetSize,
|
||||
protectedStrategy: manifest.protectedStrategy,
|
||||
verifiable: true,
|
||||
verifyRoot: verifyRoot,
|
||||
slotRoots: @slotRoots,
|
||||
|
||||
@ -25,14 +25,35 @@ type
|
||||
OnFulfillment* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
|
||||
OnSlotFilled* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises:[].}
|
||||
OnSlotFreed* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
|
||||
OnSlotReservationsFull* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
|
||||
OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises:[].}
|
||||
OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises:[].}
|
||||
OnProofSubmitted* = proc(id: SlotId) {.gcsafe, upraises:[].}
|
||||
PastStorageRequest* = object
|
||||
ProofChallenge* = array[32, byte]
|
||||
|
||||
# Marketplace events -- located here due to the Market abstraction
|
||||
MarketplaceEvent* = Event
|
||||
StorageRequested* = object of MarketplaceEvent
|
||||
requestId*: RequestId
|
||||
ask*: StorageAsk
|
||||
expiry*: UInt256
|
||||
ProofChallenge* = array[32, byte]
|
||||
SlotFilled* = object of MarketplaceEvent
|
||||
requestId* {.indexed.}: RequestId
|
||||
slotIndex*: UInt256
|
||||
SlotFreed* = object of MarketplaceEvent
|
||||
requestId* {.indexed.}: RequestId
|
||||
slotIndex*: UInt256
|
||||
SlotReservationsFull* = object of MarketplaceEvent
|
||||
requestId* {.indexed.}: RequestId
|
||||
slotIndex*: UInt256
|
||||
RequestFulfilled* = object of MarketplaceEvent
|
||||
requestId* {.indexed.}: RequestId
|
||||
RequestCancelled* = object of MarketplaceEvent
|
||||
requestId* {.indexed.}: RequestId
|
||||
RequestFailed* = object of MarketplaceEvent
|
||||
requestId* {.indexed.}: RequestId
|
||||
ProofSubmitted* = object of MarketplaceEvent
|
||||
id*: SlotId
|
||||
|
||||
method getZkeyHash*(market: Market): Future[?string] {.base, async.} =
|
||||
raiseAssert("not implemented")
|
||||
@ -144,6 +165,20 @@ method canProofBeMarkedAsMissing*(market: Market,
|
||||
period: Period): Future[bool] {.base, async.} =
|
||||
raiseAssert("not implemented")
|
||||
|
||||
method reserveSlot*(
|
||||
market: Market,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256) {.base, async.} =
|
||||
|
||||
raiseAssert("not implemented")
|
||||
|
||||
method canReserveSlot*(
|
||||
market: Market,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256): Future[bool] {.base, async.} =
|
||||
|
||||
raiseAssert("not implemented")
|
||||
|
||||
method subscribeFulfillment*(market: Market,
|
||||
callback: OnFulfillment):
|
||||
Future[Subscription] {.base, async.} =
|
||||
@ -172,6 +207,12 @@ method subscribeSlotFreed*(market: Market,
|
||||
Future[Subscription] {.base, async.} =
|
||||
raiseAssert("not implemented")
|
||||
|
||||
method subscribeSlotReservationsFull*(
|
||||
market: Market,
|
||||
callback: OnSlotReservationsFull): Future[Subscription] {.base, async.} =
|
||||
|
||||
raiseAssert("not implemented")
|
||||
|
||||
method subscribeRequestCancelled*(market: Market,
|
||||
callback: OnRequestCancelled):
|
||||
Future[Subscription] {.base, async.} =
|
||||
@ -202,7 +243,8 @@ method subscribeProofSubmission*(market: Market,
|
||||
method unsubscribe*(subscription: Subscription) {.base, async, upraises:[].} =
|
||||
raiseAssert("not implemented")
|
||||
|
||||
method queryPastStorageRequests*(market: Market,
|
||||
blocksAgo: int):
|
||||
Future[seq[PastStorageRequest]] {.base, async.} =
|
||||
method queryPastEvents*[T: MarketplaceEvent](
|
||||
market: Market,
|
||||
_: type T,
|
||||
blocksAgo: int): Future[seq[T]] {.base, async.} =
|
||||
raiseAssert("not implemented")
|
||||
|
||||
@ -14,6 +14,8 @@ push: {.upraises: [].}
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/stew/byteutils
|
||||
import pkg/serde/json
|
||||
|
||||
import ../../units
|
||||
import ../../errors
|
||||
@ -100,3 +102,18 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
|
||||
nodes.add node
|
||||
|
||||
CodexProof.init(mcodec, index.int, nleaves.int, nodes)
|
||||
|
||||
proc fromJson*(
|
||||
_: type CodexProof,
|
||||
json: JsonNode
|
||||
): ?!CodexProof =
|
||||
expectJsonKind(Cid, JString, json)
|
||||
var bytes: seq[byte]
|
||||
try:
|
||||
bytes = hexToSeqByte(json.str)
|
||||
except ValueError as err:
|
||||
return failure(err)
|
||||
|
||||
CodexProof.decode(bytes)
|
||||
|
||||
func `%`*(proof: CodexProof): JsonNode = % byteutils.toHex(proof.encode())
|
||||
|
||||
@ -240,14 +240,14 @@ proc streamEntireDataset(
|
||||
self: CodexNodeRef,
|
||||
manifest: Manifest,
|
||||
manifestCid: Cid,
|
||||
): ?!LPStream =
|
||||
): Future[?!LPStream] {.async.} =
|
||||
## Streams the contents of the entire dataset described by the manifest.
|
||||
##
|
||||
trace "Retrieving blocks from manifest", manifestCid
|
||||
|
||||
if manifest.protected:
|
||||
# Retrieve, decode and save to the local store all EС groups
|
||||
proc erasureJob(): Future[void] {.async.} =
|
||||
proc erasureJob(): Future[?!void] {.async.} =
|
||||
try:
|
||||
# Spawn an erasure decoding job
|
||||
let
|
||||
@ -257,11 +257,20 @@ proc streamEntireDataset(
|
||||
leoDecoderProvider,
|
||||
self.taskpool)
|
||||
without _ =? (await erasure.decode(manifest)), error:
|
||||
trace "Unable to erasure decode manifest", manifestCid, exc = error.msg
|
||||
except CatchableError as exc:
|
||||
trace "Exception decoding manifest", manifestCid, exc = exc.msg
|
||||
error "Unable to erasure decode manifest", manifestCid, exc = error.msg
|
||||
return failure(error)
|
||||
|
||||
asyncSpawn erasureJob()
|
||||
return success()
|
||||
# --------------------------------------------------------------------------
|
||||
# FIXME this is a HACK so that the node does not crash during the workshop.
|
||||
# We should NOT catch Defect.
|
||||
except Exception as exc:
|
||||
trace "Exception decoding manifest", manifestCid, exc = exc.msg
|
||||
return failure(exc.msg)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
if err =? (await erasureJob()).errorOption:
|
||||
return failure(err)
|
||||
|
||||
# Retrieve all blocks of the dataset sequentially from the local store or network
|
||||
trace "Creating store stream for manifest", manifestCid
|
||||
@ -283,7 +292,7 @@ proc retrieve*(
|
||||
|
||||
return await self.streamSingleBlock(cid)
|
||||
|
||||
self.streamEntireDataset(manifest, cid)
|
||||
await self.streamEntireDataset(manifest, cid)
|
||||
|
||||
proc store*(
|
||||
self: CodexNodeRef,
|
||||
@ -357,9 +366,6 @@ proc store*(
|
||||
blocks = manifest.blocksCount,
|
||||
datasetSize = manifest.datasetSize
|
||||
|
||||
await self.discovery.provide(manifestBlk.cid)
|
||||
await self.discovery.provide(treeCid)
|
||||
|
||||
return manifestBlk.cid.success
|
||||
|
||||
proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} =
|
||||
@ -414,6 +420,15 @@ proc setupRequest(
|
||||
trace "Unable to fetch manifest for cid"
|
||||
return failure error
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# FIXME this is a BAND-AID to address
|
||||
# https://github.com/codex-storage/nim-codex/issues/852 temporarily for the
|
||||
# workshop. Remove this once we get that fixed.
|
||||
if manifest.blocksCount.uint == ecK:
|
||||
return failure("Cannot setup slots for a dataset with ecK == numBlocks. Please use a larger file or a different combination of `nodes` and `tolerance`.")
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Erasure code the dataset according to provided parameters
|
||||
let
|
||||
erasure = Erasure.new(
|
||||
@ -534,7 +549,9 @@ proc onStore(
|
||||
trace "Unable to fetch manifest for cid", cid, err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without builder =? Poseidon2Builder.new(self.networkStore, manifest), err:
|
||||
without builder =? Poseidon2Builder.new(
|
||||
self.networkStore, manifest, manifest.verifiableStrategy
|
||||
), err:
|
||||
trace "Unable to create slots builder", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
@ -559,8 +576,8 @@ proc onStore(
|
||||
|
||||
return success()
|
||||
|
||||
without indexer =? manifest.protectedStrategy.init(
|
||||
0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err:
|
||||
without indexer =? manifest.verifiableStrategy.init(
|
||||
0, manifest.blocksCount - 1, manifest.numSlots).catch, err:
|
||||
trace "Unable to create indexing strategy from protected manifest", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
|
||||
@ -32,6 +32,7 @@ import ../node
|
||||
import ../blocktype
|
||||
import ../conf
|
||||
import ../contracts
|
||||
import ../erasure/erasure
|
||||
import ../manifest
|
||||
import ../streams/asyncstreamwrapper
|
||||
import ../stores
|
||||
@ -109,6 +110,20 @@ proc retrieveCid(
|
||||
proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
|
||||
|
||||
router.api(
|
||||
MethodOptions,
|
||||
"/api/codex/v1/data") do (
|
||||
resp: HttpResponseRef) -> RestApiResponse:
|
||||
|
||||
if corsOrigin =? allowedOrigin:
|
||||
resp.setHeader("Access-Control-Allow-Origin", corsOrigin)
|
||||
resp.setHeader("Access-Control-Allow-Methods", "POST, OPTIONS")
|
||||
resp.setHeader("Access-Control-Allow-Headers", "content-type")
|
||||
resp.setHeader("Access-Control-Max-Age", "86400")
|
||||
|
||||
resp.status = Http204
|
||||
await resp.sendBody("")
|
||||
|
||||
router.rawApi(
|
||||
MethodPost,
|
||||
"/api/codex/v1/data") do (
|
||||
@ -209,13 +224,15 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
|
||||
return RestApiResponse.response($json, contentType="application/json")
|
||||
|
||||
proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin
|
||||
|
||||
router.api(
|
||||
MethodGet,
|
||||
"/api/codex/v1/sales/slots") do () -> RestApiResponse:
|
||||
## Returns active slots for the host
|
||||
try:
|
||||
without contracts =? node.contracts.host:
|
||||
return RestApiResponse.error(Http503, "Sales unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
let json = %(await contracts.sales.mySlots())
|
||||
return RestApiResponse.response($json, contentType="application/json")
|
||||
@ -230,7 +247,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
## slot is not active for the host.
|
||||
|
||||
without contracts =? node.contracts.host:
|
||||
return RestApiResponse.error(Http503, "Sales unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
without slotId =? slotId.tryGet.catch, error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
@ -241,7 +258,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
let restAgent = RestSalesAgent(
|
||||
state: agent.state() |? "none",
|
||||
slotIndex: agent.data.slotIndex,
|
||||
requestId: agent.data.requestId
|
||||
requestId: agent.data.requestId,
|
||||
request: agent.data.request,
|
||||
reservation: agent.data.reservation,
|
||||
)
|
||||
|
||||
return RestApiResponse.response(restAgent.toJson, contentType="application/json")
|
||||
@ -253,7 +272,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
|
||||
try:
|
||||
without contracts =? node.contracts.host:
|
||||
return RestApiResponse.error(Http503, "Sales unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
without avails =? (await contracts.sales.context.reservations.all(Availability)), err:
|
||||
return RestApiResponse.error(Http500, err.msg)
|
||||
@ -272,25 +291,32 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
##
|
||||
## totalSize - size of available storage in bytes
|
||||
## duration - maximum time the storage should be sold for (in seconds)
|
||||
## minPrice - minimum price to be paid (in amount of tokens)
|
||||
## minPrice - minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration
|
||||
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
|
||||
|
||||
var headers = newSeq[(string,string)]()
|
||||
|
||||
if corsOrigin =? allowedOrigin:
|
||||
headers.add(("Access-Control-Allow-Origin", corsOrigin))
|
||||
headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS"))
|
||||
headers.add(("Access-Control-Max-Age", "86400"))
|
||||
|
||||
try:
|
||||
without contracts =? node.contracts.host:
|
||||
return RestApiResponse.error(Http503, "Sales unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
|
||||
|
||||
let body = await request.getBody()
|
||||
|
||||
without restAv =? RestAvailability.fromJson(body), error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
||||
|
||||
let reservations = contracts.sales.context.reservations
|
||||
|
||||
if restAv.totalSize == 0:
|
||||
return RestApiResponse.error(Http400, "Total size must be larger then zero")
|
||||
return RestApiResponse.error(Http400, "Total size must be larger then zero", headers = headers)
|
||||
|
||||
if not reservations.hasAvailable(restAv.totalSize.truncate(uint)):
|
||||
return RestApiResponse.error(Http422, "Not enough storage quota")
|
||||
return RestApiResponse.error(Http422, "Not enough storage quota", headers = headers)
|
||||
|
||||
without availability =? (
|
||||
await reservations.createAvailability(
|
||||
@ -299,14 +325,27 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
restAv.minPrice,
|
||||
restAv.maxCollateral)
|
||||
), error:
|
||||
return RestApiResponse.error(Http500, error.msg)
|
||||
return RestApiResponse.error(Http500, error.msg, headers = headers)
|
||||
|
||||
return RestApiResponse.response(availability.toJson,
|
||||
Http201,
|
||||
contentType="application/json")
|
||||
contentType="application/json",
|
||||
headers = headers)
|
||||
except CatchableError as exc:
|
||||
trace "Excepting processing request", exc = exc.msg
|
||||
return RestApiResponse.error(Http500)
|
||||
return RestApiResponse.error(Http500, headers = headers)
|
||||
|
||||
router.api(
|
||||
MethodOptions,
|
||||
"/api/codex/v1/sales/availability/{id}") do (id: AvailabilityId, resp: HttpResponseRef) -> RestApiResponse:
|
||||
|
||||
if corsOrigin =? allowedOrigin:
|
||||
resp.setHeader("Access-Control-Allow-Origin", corsOrigin)
|
||||
resp.setHeader("Access-Control-Allow-Methods", "PATCH, OPTIONS")
|
||||
resp.setHeader("Access-Control-Max-Age", "86400")
|
||||
|
||||
resp.status = Http204
|
||||
await resp.sendBody("")
|
||||
|
||||
router.rawApi(
|
||||
MethodPatch,
|
||||
@ -322,7 +361,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
|
||||
try:
|
||||
without contracts =? node.contracts.host:
|
||||
return RestApiResponse.error(Http503, "Sales unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
without id =? id.tryGet.catch, error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
@ -378,7 +417,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
|
||||
try:
|
||||
without contracts =? node.contracts.host:
|
||||
return RestApiResponse.error(Http503, "Sales unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
without id =? id.tryGet.catch, error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
@ -386,6 +425,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
|
||||
let reservations = contracts.sales.context.reservations
|
||||
let market = contracts.sales.context.market
|
||||
|
||||
if error =? (await reservations.get(keyId, Availability)).errorOption:
|
||||
if error of NotExistsError:
|
||||
@ -403,6 +443,8 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
return RestApiResponse.error(Http500)
|
||||
|
||||
proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin
|
||||
|
||||
router.rawApi(
|
||||
MethodPost,
|
||||
"/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse:
|
||||
@ -417,29 +459,47 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
## tolerance - allowed number of nodes that can be lost before content is lost
|
||||
## colateral - requested collateral from hosts when they fill slot
|
||||
|
||||
var headers = newSeq[(string,string)]()
|
||||
|
||||
if corsOrigin =? allowedOrigin:
|
||||
headers.add(("Access-Control-Allow-Origin", corsOrigin))
|
||||
headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS"))
|
||||
headers.add(("Access-Control-Max-Age", "86400"))
|
||||
|
||||
try:
|
||||
without contracts =? node.contracts.client:
|
||||
return RestApiResponse.error(Http503, "Purchasing unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
|
||||
|
||||
without cid =? cid.tryGet.catch, error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
||||
|
||||
let body = await request.getBody()
|
||||
|
||||
without params =? StorageRequestParams.fromJson(body), error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
||||
|
||||
let nodes = params.nodes |? 1
|
||||
let tolerance = params.tolerance |? 0
|
||||
let nodes = params.nodes |? 3
|
||||
let tolerance = params.tolerance |? 1
|
||||
|
||||
if (nodes - tolerance) < 1:
|
||||
return RestApiResponse.error(Http400, "Tolerance cannot be greater or equal than nodes (nodes - tolerance)")
|
||||
if tolerance == 0:
|
||||
return RestApiResponse.error(Http400, "Tolerance needs to be bigger then zero", headers = headers)
|
||||
|
||||
# prevent underflow
|
||||
if tolerance > nodes:
|
||||
return RestApiResponse.error(Http400, "Invalid parameters: `tolerance` cannot be greater than `nodes`", headers = headers)
|
||||
|
||||
let ecK = nodes - tolerance
|
||||
let ecM = tolerance # for readability
|
||||
|
||||
# ensure leopard constrainst of 1 < K ≥ M
|
||||
if ecK <= 1 or ecK < ecM:
|
||||
return RestApiResponse.error(Http400, "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`", headers = headers)
|
||||
|
||||
without expiry =? params.expiry:
|
||||
return RestApiResponse.error(Http400, "Expiry required")
|
||||
return RestApiResponse.error(Http400, "Expiry required", headers = headers)
|
||||
|
||||
if expiry <= 0 or expiry >= params.duration:
|
||||
return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration")
|
||||
return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration", headers = headers)
|
||||
|
||||
without purchaseId =? await node.requestStorage(
|
||||
cid,
|
||||
@ -451,12 +511,17 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
params.collateral,
|
||||
expiry), error:
|
||||
|
||||
return RestApiResponse.error(Http500, error.msg)
|
||||
if error of InsufficientBlocksError:
|
||||
return RestApiResponse.error(Http400,
|
||||
"Dataset too small for erasure parameters, need at least " &
|
||||
$(ref InsufficientBlocksError)(error).minSize.int & " bytes", headers = headers)
|
||||
|
||||
return RestApiResponse.error(Http500, error.msg, headers = headers)
|
||||
|
||||
return RestApiResponse.response(purchaseId.toHex)
|
||||
except CatchableError as exc:
|
||||
trace "Excepting processing request", exc = exc.msg
|
||||
return RestApiResponse.error(Http500)
|
||||
return RestApiResponse.error(Http500, headers = headers)
|
||||
|
||||
router.api(
|
||||
MethodGet,
|
||||
@ -465,7 +530,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
|
||||
try:
|
||||
without contracts =? node.contracts.client:
|
||||
return RestApiResponse.error(Http503, "Purchasing unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
without id =? id.tryGet.catch, error:
|
||||
return RestApiResponse.error(Http400, error.msg)
|
||||
@ -490,7 +555,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
||||
"/api/codex/v1/storage/purchases") do () -> RestApiResponse:
|
||||
try:
|
||||
without contracts =? node.contracts.client:
|
||||
return RestApiResponse.error(Http503, "Purchasing unavailable")
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
||||
|
||||
let purchaseIds = contracts.purchasing.getPurchaseIds()
|
||||
return RestApiResponse.response($ %purchaseIds, contentType="application/json")
|
||||
|
||||
@ -7,6 +7,7 @@ import ../sales
|
||||
import ../purchasing
|
||||
import ../utils/json
|
||||
import ../manifest
|
||||
import ../units
|
||||
|
||||
export json
|
||||
|
||||
@ -37,6 +38,8 @@ type
|
||||
state* {.serialize.}: string
|
||||
requestId* {.serialize.}: RequestId
|
||||
slotIndex* {.serialize.}: UInt256
|
||||
request* {.serialize.}: ?StorageRequest
|
||||
reservation* {.serialize.}: ?Reservation
|
||||
|
||||
RestContent* = object
|
||||
cid* {.serialize.}: Cid
|
||||
@ -65,10 +68,10 @@ type
|
||||
id*: NodeId
|
||||
|
||||
RestRepoStore* = object
|
||||
totalBlocks* {.serialize.}: uint
|
||||
quotaMaxBytes* {.serialize.}: uint
|
||||
quotaUsedBytes* {.serialize.}: uint
|
||||
quotaReservedBytes* {.serialize.}: uint
|
||||
totalBlocks* {.serialize.}: Natural
|
||||
quotaMaxBytes* {.serialize.}: NBytes
|
||||
quotaUsedBytes* {.serialize.}: NBytes
|
||||
quotaReservedBytes* {.serialize.}: NBytes
|
||||
|
||||
proc init*(_: type RestContentList, content: seq[RestContent]): RestContentList =
|
||||
RestContentList(
|
||||
|
||||
@ -180,7 +180,7 @@ proc filled(
|
||||
processing.complete()
|
||||
|
||||
proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
|
||||
debug "processing slot from queue", requestId = item.requestId,
|
||||
debug "Processing slot from queue", requestId = item.requestId,
|
||||
slot = item.slotIndex
|
||||
|
||||
let agent = newSalesAgent(
|
||||
@ -202,13 +202,17 @@ proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
|
||||
proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} =
|
||||
let reservations = sales.context.reservations
|
||||
without reservs =? await reservations.all(Reservation):
|
||||
info "no unused reservations found for deletion"
|
||||
return
|
||||
|
||||
let unused = reservs.filter(r => (
|
||||
let slotId = slotId(r.requestId, r.slotIndex)
|
||||
not activeSlots.any(slot => slot.id == slotId)
|
||||
))
|
||||
info "found unused reservations for deletion", unused = unused.len
|
||||
|
||||
if unused.len == 0:
|
||||
return
|
||||
|
||||
info "Found unused reservations for deletion", unused = unused.len
|
||||
|
||||
for reservation in unused:
|
||||
|
||||
@ -219,9 +223,9 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.}
|
||||
if err =? (await reservations.deleteReservation(
|
||||
reservation.id, reservation.availabilityId
|
||||
)).errorOption:
|
||||
error "failed to delete unused reservation", error = err.msg
|
||||
error "Failed to delete unused reservation", error = err.msg
|
||||
else:
|
||||
trace "deleted unused reservation"
|
||||
trace "Deleted unused reservation"
|
||||
|
||||
proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} =
|
||||
let market = sales.context.market
|
||||
@ -461,6 +465,23 @@ proc subscribeSlotFreed(sales: Sales) {.async.} =
|
||||
except CatchableError as e:
|
||||
error "Unable to subscribe to slot freed events", msg = e.msg
|
||||
|
||||
proc subscribeSlotReservationsFull(sales: Sales) {.async.} =
|
||||
let context = sales.context
|
||||
let market = context.market
|
||||
let queue = context.slotQueue
|
||||
|
||||
proc onSlotReservationsFull(requestId: RequestId, slotIndex: UInt256) =
|
||||
trace "reservations for slot full, removing from slot queue", requestId, slotIndex
|
||||
queue.delete(requestId, slotIndex.truncate(uint16))
|
||||
|
||||
try:
|
||||
let sub = await market.subscribeSlotReservationsFull(onSlotReservationsFull)
|
||||
sales.subscriptions.add(sub)
|
||||
except CancelledError as error:
|
||||
raise error
|
||||
except CatchableError as e:
|
||||
error "Unable to subscribe to slot filled events", msg = e.msg
|
||||
|
||||
proc startSlotQueue(sales: Sales) {.async.} =
|
||||
let slotQueue = sales.context.slotQueue
|
||||
let reservations = sales.context.reservations
|
||||
@ -484,6 +505,7 @@ proc subscribe(sales: Sales) {.async.} =
|
||||
await sales.subscribeSlotFilled()
|
||||
await sales.subscribeSlotFreed()
|
||||
await sales.subscribeCancellation()
|
||||
await sales.subscribeSlotReservationsFull()
|
||||
|
||||
proc unsubscribe(sales: Sales) {.async.} =
|
||||
for sub in sales.subscriptions:
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
## |----------------------------------------| |--------------------------------------|
|
||||
## | UInt256 | totalSize | | | UInt256 | size | |
|
||||
## |----------------------------------------| |--------------------------------------|
|
||||
## | UInt256 | freeSize | | | SlotId | slotId | |
|
||||
## | UInt256 | freeSize | | | UInt256 | slotIndex | |
|
||||
## |----------------------------------------| +--------------------------------------+
|
||||
## | UInt256 | duration | |
|
||||
## |----------------------------------------|
|
||||
@ -46,6 +46,7 @@ import ../stores
|
||||
import ../market
|
||||
import ../contracts/requests
|
||||
import ../utils/json
|
||||
import ../units
|
||||
|
||||
export requests
|
||||
export logutils
|
||||
@ -53,6 +54,7 @@ export logutils
|
||||
logScope:
|
||||
topics = "sales reservations"
|
||||
|
||||
|
||||
type
|
||||
AvailabilityId* = distinct array[32, byte]
|
||||
ReservationId* = distinct array[32, byte]
|
||||
@ -63,7 +65,7 @@ type
|
||||
totalSize* {.serialize.}: UInt256
|
||||
freeSize* {.serialize.}: UInt256
|
||||
duration* {.serialize.}: UInt256
|
||||
minPrice* {.serialize.}: UInt256
|
||||
minPrice* {.serialize.}: UInt256 # minimal price paid for the whole hosted slot for the request's duration
|
||||
maxCollateral* {.serialize.}: UInt256
|
||||
Reservation* = ref object
|
||||
id* {.serialize.}: ReservationId
|
||||
@ -71,7 +73,8 @@ type
|
||||
size* {.serialize.}: UInt256
|
||||
requestId* {.serialize.}: RequestId
|
||||
slotIndex* {.serialize.}: UInt256
|
||||
Reservations* = ref object
|
||||
Reservations* = ref object of RootObj
|
||||
availabilityLock: AsyncLock # Lock for protecting assertions of availability's sizes when searching for matching availability
|
||||
repo: RepoStore
|
||||
onAvailabilityAdded: ?OnAvailabilityAdded
|
||||
GetNext* = proc(): Future[?seq[byte]] {.upraises: [], gcsafe, closure.}
|
||||
@ -95,12 +98,22 @@ const
|
||||
SalesKey = (CodexMetaKey / "sales").tryGet # TODO: move to sales module
|
||||
ReservationsKey = (SalesKey / "reservations").tryGet
|
||||
|
||||
proc hash*(x: AvailabilityId): Hash {.borrow.}
|
||||
proc all*(self: Reservations, T: type SomeStorableObject): Future[?!seq[T]] {.async.}
|
||||
|
||||
template withLock(lock, body) =
|
||||
try:
|
||||
await lock.acquire()
|
||||
body
|
||||
finally:
|
||||
if lock.locked:
|
||||
lock.release()
|
||||
|
||||
|
||||
proc new*(T: type Reservations,
|
||||
repo: RepoStore): Reservations =
|
||||
|
||||
T(repo: repo)
|
||||
T(availabilityLock: newAsyncLock(),repo: repo)
|
||||
|
||||
proc init*(
|
||||
_: type Availability,
|
||||
@ -166,16 +179,16 @@ func key*(availability: Availability): ?!Key =
|
||||
func key*(reservation: Reservation): ?!Key =
|
||||
return key(reservation.id, reservation.availabilityId)
|
||||
|
||||
func available*(self: Reservations): uint = self.repo.available
|
||||
func available*(self: Reservations): uint = self.repo.available.uint
|
||||
|
||||
func hasAvailable*(self: Reservations, bytes: uint): bool =
|
||||
self.repo.available(bytes)
|
||||
self.repo.available(bytes.NBytes)
|
||||
|
||||
proc exists*(
|
||||
self: Reservations,
|
||||
key: Key): Future[bool] {.async.} =
|
||||
|
||||
let exists = await self.repo.metaDs.contains(key)
|
||||
let exists = await self.repo.metaDs.ds.contains(key)
|
||||
return exists
|
||||
|
||||
proc getImpl(
|
||||
@ -186,7 +199,7 @@ proc getImpl(
|
||||
let err = newException(NotExistsError, "object with key " & $key & " does not exist")
|
||||
return failure(err)
|
||||
|
||||
without serialized =? await self.repo.metaDs.get(key), error:
|
||||
without serialized =? await self.repo.metaDs.ds.get(key), error:
|
||||
return failure(error.toErr(GetFailedError))
|
||||
|
||||
return success serialized
|
||||
@ -213,7 +226,7 @@ proc updateImpl(
|
||||
without key =? obj.key, error:
|
||||
return failure(error)
|
||||
|
||||
if err =? (await self.repo.metaDs.put(
|
||||
if err =? (await self.repo.metaDs.ds.put(
|
||||
key,
|
||||
@(obj.toJson.toBytes)
|
||||
)).errorOption:
|
||||
@ -221,20 +234,19 @@ proc updateImpl(
|
||||
|
||||
return success()
|
||||
|
||||
proc update*(
|
||||
self: Reservations,
|
||||
obj: Reservation): Future[?!void] {.async.} =
|
||||
return await self.updateImpl(obj)
|
||||
|
||||
proc update*(
|
||||
proc updateAvailability(
|
||||
self: Reservations,
|
||||
obj: Availability): Future[?!void] {.async.} =
|
||||
|
||||
logScope:
|
||||
availabilityId = obj.id
|
||||
|
||||
without key =? obj.key, error:
|
||||
return failure(error)
|
||||
|
||||
without oldAvailability =? await self.get(key, Availability), err:
|
||||
if err of NotExistsError:
|
||||
trace "Creating new Availability"
|
||||
let res = await self.updateImpl(obj)
|
||||
# inform subscribers that Availability has been added
|
||||
if onAvailabilityAdded =? self.onAvailabilityAdded:
|
||||
@ -248,20 +260,20 @@ proc update*(
|
||||
except CatchableError as e:
|
||||
# we don't have any insight into types of exceptions that
|
||||
# `onAvailabilityAdded` can raise because it is caller-defined
|
||||
warn "Unknown error during 'onAvailabilityAdded' callback",
|
||||
availabilityId = obj.id, error = e.msg
|
||||
warn "Unknown error during 'onAvailabilityAdded' callback", error = e.msg
|
||||
return res
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
# Sizing of the availability changed, we need to adjust the repo reservation accordingly
|
||||
if oldAvailability.totalSize != obj.totalSize:
|
||||
trace "totalSize changed, updating repo reservation"
|
||||
if oldAvailability.totalSize < obj.totalSize: # storage added
|
||||
if reserveErr =? (await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).truncate(uint))).errorOption:
|
||||
if reserveErr =? (await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).truncate(uint).NBytes)).errorOption:
|
||||
return failure(reserveErr.toErr(ReserveFailedError))
|
||||
|
||||
elif oldAvailability.totalSize > obj.totalSize: # storage removed
|
||||
if reserveErr =? (await self.repo.release((oldAvailability.totalSize - obj.totalSize).truncate(uint))).errorOption:
|
||||
if reserveErr =? (await self.repo.release((oldAvailability.totalSize - obj.totalSize).truncate(uint).NBytes)).errorOption:
|
||||
return failure(reserveErr.toErr(ReleaseFailedError))
|
||||
|
||||
let res = await self.updateImpl(obj)
|
||||
@ -280,11 +292,21 @@ proc update*(
|
||||
except CatchableError as e:
|
||||
# we don't have any insight into types of exceptions that
|
||||
# `onAvailabilityAdded` can raise because it is caller-defined
|
||||
warn "Unknown error during 'onAvailabilityAdded' callback",
|
||||
availabilityId = obj.id, error = e.msg
|
||||
warn "Unknown error during 'onAvailabilityAdded' callback", error = e.msg
|
||||
|
||||
return res
|
||||
|
||||
proc update*(
|
||||
self: Reservations,
|
||||
obj: Reservation): Future[?!void] {.async.} =
|
||||
return await self.updateImpl(obj)
|
||||
|
||||
proc update*(
|
||||
self: Reservations,
|
||||
obj: Availability): Future[?!void] {.async.} =
|
||||
withLock(self.availabilityLock):
|
||||
return await self.updateAvailability(obj)
|
||||
|
||||
proc delete(
|
||||
self: Reservations,
|
||||
key: Key): Future[?!void] {.async.} =
|
||||
@ -294,7 +316,7 @@ proc delete(
|
||||
if not await self.exists(key):
|
||||
return success()
|
||||
|
||||
if err =? (await self.repo.metaDs.delete(key)).errorOption:
|
||||
if err =? (await self.repo.metaDs.ds.delete(key)).errorOption:
|
||||
return failure(err.toErr(DeleteFailedError))
|
||||
|
||||
return success()
|
||||
@ -312,31 +334,32 @@ proc deleteReservation*(
|
||||
without key =? key(reservationId, availabilityId), error:
|
||||
return failure(error)
|
||||
|
||||
without reservation =? (await self.get(key, Reservation)), error:
|
||||
if error of NotExistsError:
|
||||
return success()
|
||||
else:
|
||||
return failure(error)
|
||||
withLock(self.availabilityLock):
|
||||
without reservation =? (await self.get(key, Reservation)), error:
|
||||
if error of NotExistsError:
|
||||
return success()
|
||||
else:
|
||||
return failure(error)
|
||||
|
||||
if reservation.size > 0.u256:
|
||||
trace "returning remaining reservation bytes to availability",
|
||||
size = reservation.size
|
||||
if reservation.size > 0.u256:
|
||||
trace "returning remaining reservation bytes to availability",
|
||||
size = reservation.size
|
||||
|
||||
without availabilityKey =? availabilityId.key, error:
|
||||
return failure(error)
|
||||
without availabilityKey =? availabilityId.key, error:
|
||||
return failure(error)
|
||||
|
||||
without var availability =? await self.get(availabilityKey, Availability), error:
|
||||
return failure(error)
|
||||
without var availability =? await self.get(availabilityKey, Availability), error:
|
||||
return failure(error)
|
||||
|
||||
availability.freeSize += reservation.size
|
||||
availability.freeSize += reservation.size
|
||||
|
||||
if updateErr =? (await self.update(availability)).errorOption:
|
||||
return failure(updateErr)
|
||||
if updateErr =? (await self.updateAvailability(availability)).errorOption:
|
||||
return failure(updateErr)
|
||||
|
||||
if err =? (await self.repo.metaDs.delete(key)).errorOption:
|
||||
return failure(err.toErr(DeleteFailedError))
|
||||
if err =? (await self.repo.metaDs.ds.delete(key)).errorOption:
|
||||
return failure(err.toErr(DeleteFailedError))
|
||||
|
||||
return success()
|
||||
return success()
|
||||
|
||||
# TODO: add support for deleting availabilities
|
||||
# To delete, must not have any active sales.
|
||||
@ -355,14 +378,14 @@ proc createAvailability*(
|
||||
)
|
||||
let bytes = availability.freeSize.truncate(uint)
|
||||
|
||||
if reserveErr =? (await self.repo.reserve(bytes)).errorOption:
|
||||
if reserveErr =? (await self.repo.reserve(bytes.NBytes)).errorOption:
|
||||
return failure(reserveErr.toErr(ReserveFailedError))
|
||||
|
||||
if updateErr =? (await self.update(availability)).errorOption:
|
||||
|
||||
# rollback the reserve
|
||||
trace "rolling back reserve"
|
||||
if rollbackErr =? (await self.repo.release(bytes)).errorOption:
|
||||
if rollbackErr =? (await self.repo.release(bytes.NBytes)).errorOption:
|
||||
rollbackErr.parent = updateErr
|
||||
return failure(rollbackErr)
|
||||
|
||||
@ -370,54 +393,57 @@ proc createAvailability*(
|
||||
|
||||
return success(availability)
|
||||
|
||||
proc createReservation*(
|
||||
method createReservation*(
|
||||
self: Reservations,
|
||||
availabilityId: AvailabilityId,
|
||||
slotSize: UInt256,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256
|
||||
): Future[?!Reservation] {.async.} =
|
||||
): Future[?!Reservation] {.async, base.} =
|
||||
|
||||
trace "creating reservation", availabilityId, slotSize, requestId, slotIndex
|
||||
withLock(self.availabilityLock):
|
||||
without availabilityKey =? availabilityId.key, error:
|
||||
return failure(error)
|
||||
|
||||
let reservation = Reservation.init(availabilityId, slotSize, requestId, slotIndex)
|
||||
without availability =? await self.get(availabilityKey, Availability), error:
|
||||
return failure(error)
|
||||
|
||||
without availabilityKey =? availabilityId.key, error:
|
||||
return failure(error)
|
||||
# Check that the found availability has enough free space after the lock has been acquired, to prevent asynchronous Availiability modifications
|
||||
if availability.freeSize < slotSize:
|
||||
let error = newException(
|
||||
BytesOutOfBoundsError,
|
||||
"trying to reserve an amount of bytes that is greater than the total size of the Availability")
|
||||
return failure(error)
|
||||
|
||||
without var availability =? await self.get(availabilityKey, Availability), error:
|
||||
return failure(error)
|
||||
trace "Creating reservation", availabilityId, slotSize, requestId, slotIndex
|
||||
|
||||
if availability.freeSize < slotSize:
|
||||
let error = newException(
|
||||
BytesOutOfBoundsError,
|
||||
"trying to reserve an amount of bytes that is greater than the total size of the Availability")
|
||||
return failure(error)
|
||||
let reservation = Reservation.init(availabilityId, slotSize, requestId, slotIndex)
|
||||
|
||||
if createResErr =? (await self.update(reservation)).errorOption:
|
||||
return failure(createResErr)
|
||||
if createResErr =? (await self.update(reservation)).errorOption:
|
||||
return failure(createResErr)
|
||||
|
||||
# reduce availability freeSize by the slot size, which is now accounted for in
|
||||
# the newly created Reservation
|
||||
availability.freeSize -= slotSize
|
||||
# reduce availability freeSize by the slot size, which is now accounted for in
|
||||
# the newly created Reservation
|
||||
availability.freeSize -= slotSize
|
||||
|
||||
# update availability with reduced size
|
||||
if updateErr =? (await self.update(availability)).errorOption:
|
||||
# update availability with reduced size
|
||||
trace "Updating availability with reduced size"
|
||||
if updateErr =? (await self.updateAvailability(availability)).errorOption:
|
||||
trace "Updating availability failed, rolling back reservation creation"
|
||||
|
||||
trace "rolling back reservation creation"
|
||||
without key =? reservation.key, keyError:
|
||||
keyError.parent = updateErr
|
||||
return failure(keyError)
|
||||
|
||||
without key =? reservation.key, keyError:
|
||||
keyError.parent = updateErr
|
||||
return failure(keyError)
|
||||
# rollback the reservation creation
|
||||
if rollbackErr =? (await self.delete(key)).errorOption:
|
||||
rollbackErr.parent = updateErr
|
||||
return failure(rollbackErr)
|
||||
|
||||
# rollback the reservation creation
|
||||
if rollbackErr =? (await self.delete(key)).errorOption:
|
||||
rollbackErr.parent = updateErr
|
||||
return failure(rollbackErr)
|
||||
return failure(updateErr)
|
||||
|
||||
return failure(updateErr)
|
||||
|
||||
return success(reservation)
|
||||
trace "Reservation succesfully created"
|
||||
return success(reservation)
|
||||
|
||||
proc returnBytesToAvailability*(
|
||||
self: Reservations,
|
||||
@ -429,48 +455,48 @@ proc returnBytesToAvailability*(
|
||||
reservationId
|
||||
availabilityId
|
||||
|
||||
withLock(self.availabilityLock):
|
||||
without key =? key(reservationId, availabilityId), error:
|
||||
return failure(error)
|
||||
|
||||
without key =? key(reservationId, availabilityId), error:
|
||||
return failure(error)
|
||||
without var reservation =? (await self.get(key, Reservation)), error:
|
||||
return failure(error)
|
||||
|
||||
without var reservation =? (await self.get(key, Reservation)), error:
|
||||
return failure(error)
|
||||
# We are ignoring bytes that are still present in the Reservation because
|
||||
# they will be returned to Availability through `deleteReservation`.
|
||||
let bytesToBeReturned = bytes - reservation.size
|
||||
|
||||
# We are ignoring bytes that are still present in the Reservation because
|
||||
# they will be returned to Availability through `deleteReservation`.
|
||||
let bytesToBeReturned = bytes - reservation.size
|
||||
if bytesToBeReturned == 0:
|
||||
trace "No bytes are returned", requestSizeBytes = bytes, returningBytes = bytesToBeReturned
|
||||
return success()
|
||||
|
||||
trace "Returning bytes", requestSizeBytes = bytes, returningBytes = bytesToBeReturned
|
||||
|
||||
# First lets see if we can re-reserve the bytes, if the Repo's quota
|
||||
# is depleted then we will fail-fast as there is nothing to be done atm.
|
||||
if reserveErr =? (await self.repo.reserve(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
|
||||
return failure(reserveErr.toErr(ReserveFailedError))
|
||||
|
||||
without availabilityKey =? availabilityId.key, error:
|
||||
return failure(error)
|
||||
|
||||
without var availability =? await self.get(availabilityKey, Availability), error:
|
||||
return failure(error)
|
||||
|
||||
availability.freeSize += bytesToBeReturned
|
||||
|
||||
# Update availability with returned size
|
||||
if updateErr =? (await self.updateAvailability(availability)).errorOption:
|
||||
|
||||
trace "Rolling back returning bytes"
|
||||
if rollbackErr =? (await self.repo.release(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
|
||||
rollbackErr.parent = updateErr
|
||||
return failure(rollbackErr)
|
||||
|
||||
return failure(updateErr)
|
||||
|
||||
if bytesToBeReturned == 0:
|
||||
trace "No bytes are returned", requestSizeBytes = bytes, returningBytes = bytesToBeReturned
|
||||
return success()
|
||||
|
||||
trace "Returning bytes", requestSizeBytes = bytes, returningBytes = bytesToBeReturned
|
||||
|
||||
# First lets see if we can re-reserve the bytes, if the Repo's quota
|
||||
# is depleted then we will fail-fast as there is nothing to be done atm.
|
||||
if reserveErr =? (await self.repo.reserve(bytesToBeReturned.truncate(uint))).errorOption:
|
||||
return failure(reserveErr.toErr(ReserveFailedError))
|
||||
|
||||
without availabilityKey =? availabilityId.key, error:
|
||||
return failure(error)
|
||||
|
||||
without var availability =? await self.get(availabilityKey, Availability), error:
|
||||
return failure(error)
|
||||
|
||||
availability.freeSize += bytesToBeReturned
|
||||
|
||||
# Update availability with returned size
|
||||
if updateErr =? (await self.update(availability)).errorOption:
|
||||
|
||||
trace "Rolling back returning bytes"
|
||||
if rollbackErr =? (await self.repo.release(bytesToBeReturned.truncate(uint))).errorOption:
|
||||
rollbackErr.parent = updateErr
|
||||
return failure(rollbackErr)
|
||||
|
||||
return failure(updateErr)
|
||||
|
||||
return success()
|
||||
|
||||
proc release*(
|
||||
self: Reservations,
|
||||
reservationId: ReservationId,
|
||||
@ -497,7 +523,7 @@ proc release*(
|
||||
"trying to release an amount of bytes that is greater than the total size of the Reservation")
|
||||
return failure(error)
|
||||
|
||||
if releaseErr =? (await self.repo.release(bytes)).errorOption:
|
||||
if releaseErr =? (await self.repo.release(bytes.NBytes)).errorOption:
|
||||
return failure(releaseErr.toErr(ReleaseFailedError))
|
||||
|
||||
reservation.size -= bytes.u256
|
||||
@ -507,7 +533,7 @@ proc release*(
|
||||
|
||||
# rollback release if an update error encountered
|
||||
trace "rolling back release"
|
||||
if rollbackErr =? (await self.repo.reserve(bytes)).errorOption:
|
||||
if rollbackErr =? (await self.repo.reserve(bytes.NBytes)).errorOption:
|
||||
rollbackErr.parent = err
|
||||
return failure(rollbackErr)
|
||||
return failure(err)
|
||||
@ -537,7 +563,7 @@ proc storables(
|
||||
else:
|
||||
raiseAssert "unknown type"
|
||||
|
||||
without results =? await self.repo.metaDs.query(query), error:
|
||||
without results =? await self.repo.metaDs.ds.query(query), error:
|
||||
return failure(error)
|
||||
|
||||
# /sales/reservations
|
||||
@ -621,6 +647,7 @@ proc findAvailability*(
|
||||
minPrice >= availability.minPrice:
|
||||
|
||||
trace "availability matched",
|
||||
id = availability.id,
|
||||
size, availFreeSize = availability.freeSize,
|
||||
duration, availDuration = availability.duration,
|
||||
minPrice, availMinPrice = availability.minPrice,
|
||||
@ -635,8 +662,8 @@ proc findAvailability*(
|
||||
return some availability
|
||||
|
||||
trace "availability did not match",
|
||||
id = availability.id,
|
||||
size, availFreeSize = availability.freeSize,
|
||||
duration, availDuration = availability.duration,
|
||||
minPrice, availMinPrice = availability.minPrice,
|
||||
collateral, availMaxCollateral = availability.maxCollateral
|
||||
|
||||
|
||||
@ -8,8 +8,13 @@ import ./errorhandling
|
||||
logScope:
|
||||
topics = "marketplace sales ignored"
|
||||
|
||||
# Ignored slots could mean there was no availability or that the slot could
|
||||
# not be reserved.
|
||||
|
||||
type
|
||||
SaleIgnored* = ref object of ErrorHandlingState
|
||||
reprocessSlot*: bool # readd slot to queue with `seen` flag
|
||||
returnBytes*: bool # return unreleased bytes from Reservation to Availability
|
||||
|
||||
method `$`*(state: SaleIgnored): string = "SaleIgnored"
|
||||
|
||||
@ -17,7 +22,5 @@ method run*(state: SaleIgnored, machine: Machine): Future[?State] {.async.} =
|
||||
let agent = SalesAgent(machine)
|
||||
|
||||
if onCleanUp =? agent.onCleanUp:
|
||||
# Ignored slots mean there was no availability. In order to prevent small
|
||||
# availabilities from draining the queue, mark this slot as seen and re-add
|
||||
# back into the queue.
|
||||
await onCleanUp(reprocessSlot = true)
|
||||
await onCleanUp(reprocessSlot = state.reprocessSlot,
|
||||
returnBytes = state.returnBytes)
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/metrics
|
||||
|
||||
import ../../logutils
|
||||
import ../../market
|
||||
@ -10,9 +11,11 @@ import ./cancelled
|
||||
import ./failed
|
||||
import ./filled
|
||||
import ./ignored
|
||||
import ./downloading
|
||||
import ./slotreserving
|
||||
import ./errored
|
||||
|
||||
declareCounter(codex_reservations_availability_mismatch, "codex reservations availability_mismatch")
|
||||
|
||||
type
|
||||
SalePreparing* = ref object of ErrorHandlingState
|
||||
|
||||
@ -47,7 +50,7 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
|
||||
let slotId = slotId(data.requestId, data.slotIndex)
|
||||
let state = await market.slotState(slotId)
|
||||
if state != SlotState.Free:
|
||||
return some State(SaleIgnored())
|
||||
return some State(SaleIgnored(reprocessSlot: false, returnBytes: false))
|
||||
|
||||
# TODO: Once implemented, check to ensure the host is allowed to fill the slot,
|
||||
# due to the [sliding window mechanism](https://github.com/codex-storage/codex-research/blob/master/design/marketplace.md#dispersal)
|
||||
@ -66,11 +69,11 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
|
||||
request.ask.duration,
|
||||
request.ask.pricePerSlot,
|
||||
request.ask.collateral):
|
||||
debug "no availability found for request, ignoring"
|
||||
debug "No availability found for request, ignoring"
|
||||
|
||||
return some State(SaleIgnored())
|
||||
return some State(SaleIgnored(reprocessSlot: true))
|
||||
|
||||
info "availability found for request, creating reservation"
|
||||
info "Availability found for request, creating reservation"
|
||||
|
||||
without reservation =? await reservations.createReservation(
|
||||
availability.id,
|
||||
@ -78,7 +81,18 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
|
||||
request.id,
|
||||
data.slotIndex
|
||||
), error:
|
||||
trace "Creation of reservation failed"
|
||||
# Race condition:
|
||||
# reservations.findAvailability (line 64) is no guarantee. You can never know for certain that the reservation can be created until after you have it.
|
||||
# Should createReservation fail because there's no space, we proceed to SaleIgnored.
|
||||
if error of BytesOutOfBoundsError:
|
||||
# Lets monitor how often this happen and if it is often we can make it more inteligent to handle it
|
||||
codex_reservations_availability_mismatch.inc()
|
||||
return some State(SaleIgnored(reprocessSlot: true))
|
||||
|
||||
return some State(SaleErrored(error: error))
|
||||
|
||||
trace "Reservation created succesfully"
|
||||
|
||||
data.reservation = some reservation
|
||||
return some State(SaleDownloading())
|
||||
return some State(SaleSlotReserving())
|
||||
|
||||
61
codex/sales/states/slotreserving.nim
Normal file
61
codex/sales/states/slotreserving.nim
Normal file
@ -0,0 +1,61 @@
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/metrics
|
||||
|
||||
import ../../logutils
|
||||
import ../../market
|
||||
import ../salesagent
|
||||
import ../statemachine
|
||||
import ./errorhandling
|
||||
import ./cancelled
|
||||
import ./failed
|
||||
import ./filled
|
||||
import ./ignored
|
||||
import ./downloading
|
||||
import ./errored
|
||||
|
||||
type
|
||||
SaleSlotReserving* = ref object of ErrorHandlingState
|
||||
|
||||
logScope:
|
||||
topics = "marketplace sales reserving"
|
||||
|
||||
method `$`*(state: SaleSlotReserving): string = "SaleSlotReserving"
|
||||
|
||||
method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State =
|
||||
return some State(SaleCancelled())
|
||||
|
||||
method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State =
|
||||
return some State(SaleFailed())
|
||||
|
||||
method onSlotFilled*(state: SaleSlotReserving, requestId: RequestId,
|
||||
slotIndex: UInt256): ?State =
|
||||
return some State(SaleFilled())
|
||||
|
||||
method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.} =
|
||||
let agent = SalesAgent(machine)
|
||||
let data = agent.data
|
||||
let context = agent.context
|
||||
let market = context.market
|
||||
|
||||
logScope:
|
||||
requestId = data.requestId
|
||||
slotIndex = data.slotIndex
|
||||
|
||||
let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex)
|
||||
if canReserve:
|
||||
try:
|
||||
trace "Reserving slot"
|
||||
await market.reserveSlot(data.requestId, data.slotIndex)
|
||||
except MarketError as e:
|
||||
return some State( SaleErrored(error: e) )
|
||||
|
||||
trace "Slot successfully reserved"
|
||||
return some State( SaleDownloading() )
|
||||
|
||||
else:
|
||||
# do not re-add this slot to the queue, and return bytes from Reservation to
|
||||
# the Availability
|
||||
debug "Slot cannot be reserved, ignoring"
|
||||
return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) )
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import ./proofs/backends
|
||||
import ./proofs/prover
|
||||
import ./proofs/backendfactory
|
||||
|
||||
export circomcompat, prover
|
||||
export circomcompat, prover, backendfactory
|
||||
|
||||
85
codex/slots/proofs/backendfactory.nim
Normal file
85
codex/slots/proofs/backendfactory.nim
Normal file
@ -0,0 +1,85 @@
|
||||
import os
|
||||
import strutils
|
||||
import pkg/chronos
|
||||
import pkg/chronicles
|
||||
import pkg/questionable
|
||||
import pkg/confutils/defs
|
||||
import pkg/stew/io2
|
||||
import pkg/ethers
|
||||
|
||||
import ../../conf
|
||||
import ./backends
|
||||
import ./backendutils
|
||||
|
||||
proc initializeFromConfig(
|
||||
config: CodexConf,
|
||||
utils: BackendUtils): ?!AnyBackend =
|
||||
if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or
|
||||
not endsWith($config.circomR1cs, ".r1cs"):
|
||||
return failure("Circom R1CS file not accessible")
|
||||
|
||||
if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or
|
||||
not endsWith($config.circomWasm, ".wasm"):
|
||||
return failure("Circom wasm file not accessible")
|
||||
|
||||
if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or
|
||||
not endsWith($config.circomZkey, ".zkey"):
|
||||
return failure("Circom zkey file not accessible")
|
||||
|
||||
trace "Initialized prover backend from cli config"
|
||||
success(utils.initializeCircomBackend(
|
||||
$config.circomR1cs,
|
||||
$config.circomWasm,
|
||||
$config.circomZkey))
|
||||
|
||||
proc r1csFilePath(config: CodexConf): string =
|
||||
config.circuitDir / "proof_main.r1cs"
|
||||
|
||||
proc wasmFilePath(config: CodexConf): string =
|
||||
config.circuitDir / "proof_main.wasm"
|
||||
|
||||
proc zkeyFilePath(config: CodexConf): string =
|
||||
config.circuitDir / "proof_main.zkey"
|
||||
|
||||
proc initializeFromCircuitDirFiles(
|
||||
config: CodexConf,
|
||||
utils: BackendUtils): ?!AnyBackend =
|
||||
if fileExists(config.r1csFilePath) and
|
||||
fileExists(config.wasmFilePath) and
|
||||
fileExists(config.zkeyFilePath):
|
||||
trace "Initialized prover backend from local files"
|
||||
return success(utils.initializeCircomBackend(
|
||||
config.r1csFilePath,
|
||||
config.wasmFilePath,
|
||||
config.zkeyFilePath))
|
||||
|
||||
failure("Circuit files not found")
|
||||
|
||||
proc suggestDownloadTool(config: CodexConf) =
|
||||
without address =? config.marketplaceAddress:
|
||||
raise (ref Defect)(msg: "Proving backend initializing while marketplace address not set.")
|
||||
|
||||
let
|
||||
tokens = [
|
||||
"cirdl",
|
||||
"\"" & $config.circuitDir & "\"",
|
||||
config.ethProvider,
|
||||
$address
|
||||
]
|
||||
instructions = "'./" & tokens.join(" ") & "'"
|
||||
|
||||
warn "Proving circuit files are not found. Please run the following to download them:", instructions
|
||||
|
||||
proc initializeBackend*(
|
||||
config: CodexConf,
|
||||
utils: BackendUtils = BackendUtils()): ?!AnyBackend =
|
||||
|
||||
without backend =? initializeFromConfig(config, utils), cliErr:
|
||||
info "Could not initialize prover backend from CLI options...", msg = cliErr.msg
|
||||
without backend =? initializeFromCircuitDirFiles(config, utils), localErr:
|
||||
info "Could not initialize prover backend from circuit dir files...", msg = localErr.msg
|
||||
suggestDownloadTool(config)
|
||||
return failure("CircuitFilesNotFound")
|
||||
# Unexpected: value of backend does not survive leaving each scope. (definition does though...)
|
||||
return success(backend)
|
||||
return success(backend)
|
||||
@ -1,3 +1,6 @@
|
||||
import ./backends/circomcompat
|
||||
|
||||
export circomcompat
|
||||
|
||||
type
|
||||
AnyBackend* = CircomCompat
|
||||
|
||||
@ -9,17 +9,14 @@
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/questionable/results
|
||||
import pkg/circomcompat
|
||||
import pkg/poseidon2/io
|
||||
|
||||
import ../../types
|
||||
import ../../../stores
|
||||
import ../../../merkletree
|
||||
import ../../../codextypes
|
||||
import ../../../contracts
|
||||
|
||||
import ./converters
|
||||
@ -39,6 +36,41 @@ type
|
||||
backendCfg : ptr CircomBn254Cfg
|
||||
vkp* : ptr CircomKey
|
||||
|
||||
NormalizedProofInputs*[H] {.borrow: `.`.} = distinct ProofInputs[H]
|
||||
|
||||
func normalizeInput*[H](self: CircomCompat, input: ProofInputs[H]):
|
||||
NormalizedProofInputs[H] =
|
||||
## Parameters in CIRCOM circuits are statically sized and must be properly
|
||||
## padded before they can be passed onto the circuit. This function takes
|
||||
## variable length parameters and performs that padding.
|
||||
##
|
||||
## The output from this function can be JSON-serialized and used as direct
|
||||
## inputs to the CIRCOM circuit for testing and debugging when one wishes
|
||||
## to bypass the Rust FFI.
|
||||
|
||||
let normSamples = collect:
|
||||
for sample in input.samples:
|
||||
var merklePaths = sample.merklePaths
|
||||
merklePaths.setLen(self.slotDepth)
|
||||
Sample[H](
|
||||
cellData: sample.cellData,
|
||||
merklePaths: merklePaths
|
||||
)
|
||||
|
||||
var normSlotProof = input.slotProof
|
||||
normSlotProof.setLen(self.datasetDepth)
|
||||
|
||||
NormalizedProofInputs[H] ProofInputs[H](
|
||||
entropy: input.entropy,
|
||||
datasetRoot: input.datasetRoot,
|
||||
slotIndex: input.slotIndex,
|
||||
slotRoot: input.slotRoot,
|
||||
nCellsPerSlot: input.nCellsPerSlot,
|
||||
nSlotsPerDataSet: input.nSlotsPerDataSet,
|
||||
slotProof: normSlotProof,
|
||||
samples: normSamples
|
||||
)
|
||||
|
||||
proc release*(self: CircomCompat) =
|
||||
## Release the ctx
|
||||
##
|
||||
@ -49,27 +81,20 @@ proc release*(self: CircomCompat) =
|
||||
if not isNil(self.vkp):
|
||||
self.vkp.unsafeAddr.release_key()
|
||||
|
||||
proc prove*[H](
|
||||
proc prove[H](
|
||||
self: CircomCompat,
|
||||
input: ProofInputs[H]): ?!CircomProof =
|
||||
## Encode buffers using a ctx
|
||||
##
|
||||
input: NormalizedProofInputs[H]): ?!CircomProof =
|
||||
|
||||
# NOTE: All inputs are statically sized per circuit
|
||||
# and adjusted accordingly right before being passed
|
||||
# to the circom ffi - `setLen` is used to adjust the
|
||||
# sequence length to the correct size which also 0 pads
|
||||
# to the correct length
|
||||
doAssert input.samples.len == self.numSamples,
|
||||
"Number of samples does not match"
|
||||
|
||||
doAssert input.slotProof.len <= self.datasetDepth,
|
||||
"Number of slot proofs does not match"
|
||||
"Slot proof is too deep - dataset has more slots than what we can handle?"
|
||||
|
||||
doAssert input.samples.allIt(
|
||||
block:
|
||||
(it.merklePaths.len <= self.slotDepth + self.blkDepth and
|
||||
it.cellData.len <= self.cellElms * 32)), "Merkle paths length does not match"
|
||||
it.cellData.len == self.cellElms)), "Merkle paths too deep or cells too big for circuit"
|
||||
|
||||
# TODO: All parameters should match circom's static parametter
|
||||
var
|
||||
@ -116,8 +141,7 @@ proc prove*[H](
|
||||
var
|
||||
slotProof = input.slotProof.mapIt( it.toBytes ).concat
|
||||
|
||||
slotProof.setLen(self.datasetDepth) # zero pad inputs to correct size
|
||||
|
||||
doAssert(slotProof.len == self.datasetDepth)
|
||||
# arrays are always flattened
|
||||
if ctx.pushInputU256Array(
|
||||
"slotProof".cstring,
|
||||
@ -128,16 +152,14 @@ proc prove*[H](
|
||||
for s in input.samples:
|
||||
var
|
||||
merklePaths = s.merklePaths.mapIt( it.toBytes )
|
||||
data = s.cellData
|
||||
data = s.cellData.mapIt( @(it.toBytes) ).concat
|
||||
|
||||
merklePaths.setLen(self.slotDepth) # zero pad inputs to correct size
|
||||
if ctx.pushInputU256Array(
|
||||
"merklePaths".cstring,
|
||||
merklePaths[0].addr,
|
||||
uint (merklePaths[0].len * merklePaths.len)) != ERR_OK:
|
||||
return failure("Failed to push merkle paths")
|
||||
|
||||
data.setLen(self.cellElms * 32) # zero pad inputs to correct size
|
||||
if ctx.pushInputU256Array(
|
||||
"cellData".cstring,
|
||||
data[0].addr,
|
||||
@ -162,6 +184,12 @@ proc prove*[H](
|
||||
|
||||
success proof
|
||||
|
||||
proc prove*[H](
|
||||
self: CircomCompat,
|
||||
input: ProofInputs[H]): ?!CircomProof =
|
||||
|
||||
self.prove(self.normalizeInput(input))
|
||||
|
||||
proc verify*[H](
|
||||
self: CircomCompat,
|
||||
proof: CircomProof,
|
||||
|
||||
12
codex/slots/proofs/backendutils.nim
Normal file
12
codex/slots/proofs/backendutils.nim
Normal file
@ -0,0 +1,12 @@
|
||||
import ./backends
|
||||
|
||||
type
|
||||
BackendUtils* = ref object of RootObj
|
||||
|
||||
method initializeCircomBackend*(
|
||||
self: BackendUtils,
|
||||
r1csFile: string,
|
||||
wasmFile: string,
|
||||
zKeyFile: string
|
||||
): AnyBackend {.base.} =
|
||||
CircomCompat.init(r1csFile, wasmFile, zKeyFile)
|
||||
@ -21,11 +21,13 @@ import ../../merkletree
|
||||
import ../../stores
|
||||
import ../../market
|
||||
import ../../utils/poseidon2digest
|
||||
import ../../conf
|
||||
|
||||
import ../builder
|
||||
import ../sampler
|
||||
|
||||
import ./backends
|
||||
import ./backendfactory
|
||||
import ../types
|
||||
|
||||
export backends
|
||||
@ -34,7 +36,6 @@ logScope:
|
||||
topics = "codex prover"
|
||||
|
||||
type
|
||||
AnyBackend* = CircomCompat
|
||||
AnyProof* = CircomProof
|
||||
|
||||
AnySampler* = Poseidon2Sampler
|
||||
@ -86,7 +87,6 @@ proc verify*(
|
||||
inputs: AnyProofInputs): Future[?!bool] {.async.} =
|
||||
## Prove a statement using backend.
|
||||
## Returns a future that resolves to a proof.
|
||||
|
||||
self.backend.verify(proof, inputs)
|
||||
|
||||
proc new*(
|
||||
@ -96,6 +96,6 @@ proc new*(
|
||||
nSamples: int): Prover =
|
||||
|
||||
Prover(
|
||||
backend: backend,
|
||||
store: store,
|
||||
backend: backend,
|
||||
nSamples: nSamples)
|
||||
|
||||
@ -38,7 +38,7 @@ type
|
||||
func getCell*[T, H](
|
||||
self: DataSampler[T, H],
|
||||
blkBytes: seq[byte],
|
||||
blkCellIdx: Natural): seq[byte] =
|
||||
blkCellIdx: Natural): seq[H] =
|
||||
|
||||
let
|
||||
cellSize = self.builder.cellSize.uint64
|
||||
@ -47,7 +47,7 @@ func getCell*[T, H](
|
||||
|
||||
doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size"
|
||||
|
||||
toInputData[H](blkBytes[dataStart ..< dataEnd])
|
||||
blkBytes[dataStart ..< dataEnd].elements(H).toSeq()
|
||||
|
||||
proc getSample*[T, H](
|
||||
self: DataSampler[T, H],
|
||||
|
||||
@ -7,23 +7,13 @@
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import std/sugar
|
||||
import std/bitops
|
||||
import std/sequtils
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
|
||||
import pkg/constantine/math/arithmetic
|
||||
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../../merkletree
|
||||
|
||||
func toInputData*[H](data: seq[byte]): seq[byte] =
|
||||
return toSeq(data.elements(H)).mapIt( @(it.toBytes) ).concat
|
||||
|
||||
func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 =
|
||||
doAssert( k > 0 and k <= 64 )
|
||||
var r = 0'u64
|
||||
@ -39,6 +29,7 @@ func extractLowBits(fld: Poseidon2Hash, k: int): uint64 =
|
||||
return extractLowBits(elm, k);
|
||||
|
||||
func floorLog2*(x : int) : int =
|
||||
doAssert ( x > 0 )
|
||||
var k = -1
|
||||
var y = x
|
||||
while (y > 0):
|
||||
@ -47,10 +38,8 @@ func floorLog2*(x : int) : int =
|
||||
return k
|
||||
|
||||
func ceilingLog2*(x : int) : int =
|
||||
if (x == 0):
|
||||
return -1
|
||||
else:
|
||||
return (floorLog2(x-1) + 1)
|
||||
doAssert ( x > 0 )
|
||||
return (floorLog2(x - 1) + 1)
|
||||
|
||||
func toBlkInSlot*(cell: Natural, numCells: Natural): Natural =
|
||||
let log2 = ceilingLog2(numCells)
|
||||
@ -80,7 +69,7 @@ func cellIndices*(
|
||||
numCells: Natural, nSamples: Natural): seq[Natural] =
|
||||
|
||||
var indices: seq[Natural]
|
||||
while (indices.len < nSamples):
|
||||
let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1)
|
||||
indices.add(idx.Natural)
|
||||
for i in 1..nSamples:
|
||||
indices.add(cellIndex(entropy, slotRoot, numCells, i))
|
||||
|
||||
indices
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
|
||||
type
|
||||
Sample*[H] = object
|
||||
cellData*: seq[byte]
|
||||
cellData*: seq[H]
|
||||
merklePaths*: seq[H]
|
||||
|
||||
PublicInputs*[H] = object
|
||||
@ -24,5 +24,5 @@ type
|
||||
slotRoot*: H
|
||||
nCellsPerSlot*: Natural
|
||||
nSlotsPerDataSet*: Natural
|
||||
slotProof*: seq[H]
|
||||
samples*: seq[Sample[H]]
|
||||
slotProof*: seq[H] # inclusion proof that shows that the slot root (leaf) is part of the dataset (root)
|
||||
samples*: seq[Sample[H]] # inclusion proofs which show that the selected cells (leafs) are part of the slot (roots)
|
||||
|
||||
@ -29,7 +29,9 @@ type
|
||||
BlockType* {.pure.} = enum
|
||||
Manifest, Block, Both
|
||||
|
||||
CidCallback* = proc(cid: Cid): Future[void] {.gcsafe, raises:[].}
|
||||
BlockStore* = ref object of RootObj
|
||||
onBlockStored*: ?CidCallback
|
||||
|
||||
method getBlock*(self: BlockStore, cid: Cid): Future[?!Block] {.base.} =
|
||||
## Get a block from the blockstore
|
||||
|
||||
@ -197,6 +197,9 @@ method putBlock*(
|
||||
return success()
|
||||
|
||||
discard self.putBlockSync(blk)
|
||||
if onBlock =? self.onBlockStored:
|
||||
await onBlock(blk.cid)
|
||||
|
||||
return success()
|
||||
|
||||
method putCidAndProof*(
|
||||
@ -282,7 +285,8 @@ proc new*(
|
||||
cache: cache,
|
||||
cidAndProofCache: cidAndProofCache,
|
||||
currentSize: currentSize,
|
||||
size: cacheSize)
|
||||
size: cacheSize,
|
||||
onBlockStored: CidCallback.none)
|
||||
|
||||
for blk in blocks:
|
||||
discard store.putBlockSync(blk)
|
||||
|
||||
@ -59,7 +59,7 @@ proc deleteExpiredBlock(self: BlockMaintainer, cid: Cid): Future[void] {.async.}
|
||||
trace "Unable to delete block from repoStore"
|
||||
|
||||
proc processBlockExpiration(self: BlockMaintainer, be: BlockExpiration): Future[void] {.async} =
|
||||
if be.expiration < self.clock.now:
|
||||
if be.expiry < self.clock.now:
|
||||
await self.deleteExpiredBlock(be.cid)
|
||||
else:
|
||||
inc self.offset
|
||||
@ -75,11 +75,11 @@ proc runBlockCheck(self: BlockMaintainer): Future[void] {.async.} =
|
||||
return
|
||||
|
||||
var numberReceived = 0
|
||||
for maybeBeFuture in iter:
|
||||
if be =? await maybeBeFuture:
|
||||
inc numberReceived
|
||||
await self.processBlockExpiration(be)
|
||||
await sleepAsync(50.millis)
|
||||
for beFut in iter:
|
||||
let be = await beFut
|
||||
inc numberReceived
|
||||
await self.processBlockExpiration(be)
|
||||
await sleepAsync(1.millis) # cooperative scheduling
|
||||
|
||||
# If we received fewer blockExpirations from the iterator than we asked for,
|
||||
# We're at the end of the dataset and should start from 0 next time.
|
||||
|
||||
@ -6,7 +6,7 @@ import pkg/datastore/typedds
|
||||
|
||||
import ../utils/asynciter
|
||||
|
||||
type KeyVal[T] = tuple[key: Key, value: T]
|
||||
type KeyVal*[T] = tuple[key: Key, value: T]
|
||||
|
||||
proc toAsyncIter*[T](
|
||||
queryIter: QueryIter[T],
|
||||
|
||||
@ -1,679 +1,5 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2022 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
import ./repostore/store
|
||||
import ./repostore/types
|
||||
import ./repostore/coders
|
||||
|
||||
import pkg/upraises
|
||||
|
||||
push: {.upraises: [].}
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/chronos/futures
|
||||
import pkg/libp2p/[cid, multicodec, multihash]
|
||||
import pkg/lrucache
|
||||
import pkg/metrics
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/datastore
|
||||
import pkg/stew/endians2
|
||||
|
||||
import ./blockstore
|
||||
import ./keyutils
|
||||
import ../blocktype
|
||||
import ../clock
|
||||
import ../systemclock
|
||||
import ../logutils
|
||||
import ../merkletree
|
||||
import ../utils
|
||||
|
||||
export blocktype, cid
|
||||
|
||||
logScope:
|
||||
topics = "codex repostore"
|
||||
|
||||
declareGauge(codex_repostore_blocks, "codex repostore blocks")
|
||||
declareGauge(codex_repostore_bytes_used, "codex repostore bytes used")
|
||||
declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved")
|
||||
|
||||
const
|
||||
DefaultBlockTtl* = 24.hours
|
||||
DefaultQuotaBytes* = 1'u shl 33'u # ~8GB
|
||||
|
||||
type
|
||||
QuotaUsedError* = object of CodexError
|
||||
QuotaNotEnoughError* = object of CodexError
|
||||
|
||||
RepoStore* = ref object of BlockStore
|
||||
postFixLen*: int
|
||||
repoDs*: Datastore
|
||||
metaDs*: Datastore
|
||||
clock: Clock
|
||||
totalBlocks*: uint # number of blocks in the store
|
||||
quotaMaxBytes*: uint # maximum available bytes
|
||||
quotaUsedBytes*: uint # bytes used by the repo
|
||||
quotaReservedBytes*: uint # bytes reserved by the repo
|
||||
blockTtl*: Duration
|
||||
started*: bool
|
||||
|
||||
BlockExpiration* = object
|
||||
cid*: Cid
|
||||
expiration*: SecondsSince1970
|
||||
|
||||
proc updateMetrics(self: RepoStore) =
|
||||
codex_repostore_blocks.set(self.totalBlocks.int64)
|
||||
codex_repostore_bytes_used.set(self.quotaUsedBytes.int64)
|
||||
codex_repostore_bytes_reserved.set(self.quotaReservedBytes.int64)
|
||||
|
||||
func totalUsed*(self: RepoStore): uint =
|
||||
(self.quotaUsedBytes + self.quotaReservedBytes)
|
||||
|
||||
func available*(self: RepoStore): uint =
|
||||
return self.quotaMaxBytes - self.totalUsed
|
||||
|
||||
func available*(self: RepoStore, bytes: uint): bool =
|
||||
return bytes < self.available()
|
||||
|
||||
proc encode(cidAndProof: (Cid, CodexProof)): seq[byte] =
|
||||
## Encodes a tuple of cid and merkle proof in a following format:
|
||||
## | 8-bytes | n-bytes | remaining bytes |
|
||||
## | n | cid | proof |
|
||||
##
|
||||
## where n is a size of cid
|
||||
##
|
||||
let
|
||||
(cid, proof) = cidAndProof
|
||||
cidBytes = cid.data.buffer
|
||||
proofBytes = proof.encode
|
||||
n = cidBytes.len
|
||||
nBytes = n.uint64.toBytesBE
|
||||
|
||||
@nBytes & cidBytes & proofBytes
|
||||
|
||||
proc decode(_: type (Cid, CodexProof), data: seq[byte]): ?!(Cid, CodexProof) =
|
||||
let
|
||||
n = uint64.fromBytesBE(data[0..<sizeof(uint64)]).int
|
||||
cid = ? Cid.init(data[sizeof(uint64)..<sizeof(uint64) + n]).mapFailure
|
||||
proof = ? CodexProof.decode(data[sizeof(uint64) + n..^1])
|
||||
success((cid, proof))
|
||||
|
||||
proc decodeCid(_: type (Cid, CodexProof), data: seq[byte]): ?!Cid =
|
||||
let
|
||||
n = uint64.fromBytesBE(data[0..<sizeof(uint64)]).int
|
||||
cid = ? Cid.init(data[sizeof(uint64)..<sizeof(uint64) + n]).mapFailure
|
||||
success(cid)
|
||||
|
||||
method putCidAndProof*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blockCid: Cid,
|
||||
proof: CodexProof
|
||||
): Future[?!void] {.async.} =
|
||||
## Put a block to the blockstore
|
||||
##
|
||||
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
trace "Storing block cid and proof", blockCid, key
|
||||
|
||||
let value = (blockCid, proof).encode()
|
||||
|
||||
await self.metaDs.put(key, value)
|
||||
|
||||
method getCidAndProof*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural): Future[?!(Cid, CodexProof)] {.async.} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
without value =? await self.metaDs.get(key), err:
|
||||
if err of DatastoreKeyNotFound:
|
||||
return failure(newException(BlockNotFoundError, err.msg))
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
without (cid, proof) =? (Cid, CodexProof).decode(value), err:
|
||||
error "Unable to decode cid and proof", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
return success (cid, proof)
|
||||
|
||||
method getCid*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural): Future[?!Cid] {.async.} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
without value =? await self.metaDs.get(key), err:
|
||||
if err of DatastoreKeyNotFound:
|
||||
# This failure is expected to happen frequently:
|
||||
# NetworkStore.getBlock will call RepoStore.getBlock before starting the block exchange engine.
|
||||
return failure(newException(BlockNotFoundError, err.msg))
|
||||
else:
|
||||
error "Error getting cid from datastore", err = err.msg, key
|
||||
return failure(err)
|
||||
|
||||
return (Cid, CodexProof).decodeCid(value)
|
||||
|
||||
method getBlock*(self: RepoStore, cid: Cid): Future[?!Block] {.async.} =
|
||||
## Get a block from the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
if cid.isEmpty:
|
||||
return cid.emptyBlock
|
||||
|
||||
without key =? makePrefixKey(self.postFixLen, cid), err:
|
||||
error "Error getting key from provider", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without data =? await self.repoDs.get(key), err:
|
||||
if not (err of DatastoreKeyNotFound):
|
||||
error "Error getting block from datastore", err = err.msg, key
|
||||
return failure(err)
|
||||
|
||||
return failure(newException(BlockNotFoundError, err.msg))
|
||||
|
||||
return Block.new(cid, data, verify = true)
|
||||
|
||||
|
||||
method getBlockAndProof*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.async.} =
|
||||
without cidAndProof =? await self.getCidAndProof(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
let (cid, proof) = cidAndProof
|
||||
|
||||
without blk =? await self.getBlock(cid), err:
|
||||
return failure(err)
|
||||
|
||||
success((blk, proof))
|
||||
|
||||
method getBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!Block] {.async.} =
|
||||
without cid =? await self.getCid(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
await self.getBlock(cid)
|
||||
|
||||
method getBlock*(self: RepoStore, address: BlockAddress): Future[?!Block] =
|
||||
## Get a block from the blockstore
|
||||
##
|
||||
|
||||
if address.leaf:
|
||||
self.getBlock(address.treeCid, address.index)
|
||||
else:
|
||||
self.getBlock(address.cid)
|
||||
|
||||
proc getBlockExpirationEntry(
|
||||
self: RepoStore,
|
||||
cid: Cid,
|
||||
ttl: SecondsSince1970): ?!BatchEntry =
|
||||
## Get an expiration entry for a batch with timestamp
|
||||
##
|
||||
|
||||
without key =? createBlockExpirationMetadataKey(cid), err:
|
||||
return failure(err)
|
||||
|
||||
return success((key, ttl.toBytes))
|
||||
|
||||
proc getBlockExpirationEntry(
|
||||
self: RepoStore,
|
||||
cid: Cid,
|
||||
ttl: ?Duration): ?!BatchEntry =
|
||||
## Get an expiration entry for a batch for duration since "now"
|
||||
##
|
||||
|
||||
let duration = ttl |? self.blockTtl
|
||||
self.getBlockExpirationEntry(cid, self.clock.now() + duration.seconds)
|
||||
|
||||
method ensureExpiry*(
|
||||
self: RepoStore,
|
||||
cid: Cid,
|
||||
expiry: SecondsSince1970
|
||||
): Future[?!void] {.async.} =
|
||||
## Ensure that block's associated expiry is at least given timestamp
|
||||
## If the current expiry is lower then it is updated to the given one, otherwise it is left intact
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
if expiry <= 0:
|
||||
return failure(newException(ValueError, "Expiry timestamp must be larger then zero"))
|
||||
|
||||
without expiryKey =? createBlockExpirationMetadataKey(cid), err:
|
||||
return failure(err)
|
||||
|
||||
without currentExpiry =? await self.metaDs.get(expiryKey), err:
|
||||
if err of DatastoreKeyNotFound:
|
||||
error "No current expiry exists for the block"
|
||||
return failure(newException(BlockNotFoundError, err.msg))
|
||||
else:
|
||||
error "Could not read datastore key", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
logScope:
|
||||
current = currentExpiry.toSecondsSince1970
|
||||
ensuring = expiry
|
||||
|
||||
if expiry <= currentExpiry.toSecondsSince1970:
|
||||
trace "Expiry is larger than or equal to requested"
|
||||
return success()
|
||||
|
||||
if err =? (await self.metaDs.put(expiryKey, expiry.toBytes)).errorOption:
|
||||
trace "Error updating expiration metadata entry", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
return success()
|
||||
|
||||
method ensureExpiry*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
expiry: SecondsSince1970
|
||||
): Future[?!void] {.async.} =
|
||||
## Ensure that block's associated expiry is at least given timestamp
|
||||
## If the current expiry is lower then it is updated to the given one, otherwise it is left intact
|
||||
##
|
||||
without cidAndProof =? await self.getCidAndProof(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
await self.ensureExpiry(cidAndProof[0], expiry)
|
||||
|
||||
proc persistTotalBlocksCount(self: RepoStore): Future[?!void] {.async.} =
|
||||
if err =? (await self.metaDs.put(
|
||||
CodexTotalBlocksKey,
|
||||
@(self.totalBlocks.uint64.toBytesBE))).errorOption:
|
||||
trace "Error total blocks key!", err = err.msg
|
||||
return failure(err)
|
||||
return success()
|
||||
|
||||
method putBlock*(
|
||||
self: RepoStore,
|
||||
blk: Block,
|
||||
ttl = Duration.none): Future[?!void] {.async.} =
|
||||
## Put a block to the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = blk.cid
|
||||
|
||||
if blk.isEmpty:
|
||||
trace "Empty block, ignoring"
|
||||
return success()
|
||||
|
||||
without key =? makePrefixKey(self.postFixLen, blk.cid), err:
|
||||
warn "Error getting key from provider", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
if await key in self.repoDs:
|
||||
trace "Block already in store", cid = blk.cid
|
||||
return success()
|
||||
|
||||
if (self.totalUsed + blk.data.len.uint) > self.quotaMaxBytes:
|
||||
error "Cannot store block, quota used!", used = self.totalUsed
|
||||
return failure(
|
||||
newException(QuotaUsedError, "Cannot store block, quota used!"))
|
||||
|
||||
var
|
||||
batch: seq[BatchEntry]
|
||||
|
||||
let
|
||||
used = self.quotaUsedBytes + blk.data.len.uint
|
||||
|
||||
if err =? (await self.repoDs.put(key, blk.data)).errorOption:
|
||||
error "Error storing block", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
batch.add((QuotaUsedKey, @(used.uint64.toBytesBE)))
|
||||
|
||||
without blockExpEntry =? self.getBlockExpirationEntry(blk.cid, ttl), err:
|
||||
warn "Unable to create block expiration metadata key", err = err.msg
|
||||
return failure(err)
|
||||
batch.add(blockExpEntry)
|
||||
|
||||
if err =? (await self.metaDs.put(batch)).errorOption:
|
||||
error "Error updating quota bytes", err = err.msg
|
||||
|
||||
if err =? (await self.repoDs.delete(key)).errorOption:
|
||||
error "Error deleting block after failed quota update", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
return failure(err)
|
||||
|
||||
self.quotaUsedBytes = used
|
||||
inc self.totalBlocks
|
||||
if isErr (await self.persistTotalBlocksCount()):
|
||||
warn "Unable to update block total metadata"
|
||||
return failure("Unable to update block total metadata")
|
||||
|
||||
self.updateMetrics()
|
||||
return success()
|
||||
|
||||
proc updateQuotaBytesUsed(self: RepoStore, blk: Block): Future[?!void] {.async.} =
|
||||
let used = self.quotaUsedBytes - blk.data.len.uint
|
||||
if err =? (await self.metaDs.put(
|
||||
QuotaUsedKey,
|
||||
@(used.uint64.toBytesBE))).errorOption:
|
||||
trace "Error updating quota key!", err = err.msg
|
||||
return failure(err)
|
||||
self.quotaUsedBytes = used
|
||||
self.updateMetrics()
|
||||
return success()
|
||||
|
||||
proc removeBlockExpirationEntry(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
|
||||
without key =? createBlockExpirationMetadataKey(cid), err:
|
||||
return failure(err)
|
||||
return await self.metaDs.delete(key)
|
||||
|
||||
method delBlock*(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
|
||||
## Delete a block from the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
trace "Deleting block"
|
||||
|
||||
if cid.isEmpty:
|
||||
trace "Empty block, ignoring"
|
||||
return success()
|
||||
|
||||
if blk =? (await self.getBlock(cid)):
|
||||
if key =? makePrefixKey(self.postFixLen, cid) and
|
||||
err =? (await self.repoDs.delete(key)).errorOption:
|
||||
trace "Error deleting block!", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
if isErr (await self.updateQuotaBytesUsed(blk)):
|
||||
trace "Unable to update quote-bytes-used in metadata store"
|
||||
return failure("Unable to update quote-bytes-used in metadata store")
|
||||
|
||||
if isErr (await self.removeBlockExpirationEntry(blk.cid)):
|
||||
trace "Unable to remove block expiration entry from metadata store"
|
||||
return failure("Unable to remove block expiration entry from metadata store")
|
||||
|
||||
trace "Deleted block", cid, totalUsed = self.totalUsed
|
||||
|
||||
dec self.totalBlocks
|
||||
if isErr (await self.persistTotalBlocksCount()):
|
||||
trace "Unable to update block total metadata"
|
||||
return failure("Unable to update block total metadata")
|
||||
|
||||
self.updateMetrics()
|
||||
return success()
|
||||
|
||||
method delBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!void] {.async.} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
trace "Fetching proof", key
|
||||
without value =? await self.metaDs.get(key), err:
|
||||
if err of DatastoreKeyNotFound:
|
||||
return success()
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
without cid =? (Cid, CodexProof).decodeCid(value), err:
|
||||
return failure(err)
|
||||
|
||||
trace "Deleting block", cid
|
||||
if err =? (await self.delBlock(cid)).errorOption:
|
||||
return failure(err)
|
||||
|
||||
await self.metaDs.delete(key)
|
||||
|
||||
method hasBlock*(self: RepoStore, cid: Cid): Future[?!bool] {.async.} =
|
||||
## Check if the block exists in the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
if cid.isEmpty:
|
||||
trace "Empty block, ignoring"
|
||||
return success true
|
||||
|
||||
without key =? makePrefixKey(self.postFixLen, cid), err:
|
||||
trace "Error getting key from provider", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
return await self.repoDs.has(key)
|
||||
|
||||
method hasBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!bool] {.async.} =
|
||||
without cid =? await self.getCid(treeCid, index), err:
|
||||
if err of BlockNotFoundError:
|
||||
return success(false)
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
await self.hasBlock(cid)
|
||||
|
||||
method listBlocks*(
|
||||
self: RepoStore,
|
||||
blockType = BlockType.Manifest
|
||||
): Future[?!AsyncIter[?Cid]] {.async.} =
|
||||
## Get the list of blocks in the RepoStore.
|
||||
## This is an intensive operation
|
||||
##
|
||||
|
||||
var
|
||||
iter = AsyncIter[?Cid]()
|
||||
|
||||
let key =
|
||||
case blockType:
|
||||
of BlockType.Manifest: CodexManifestKey
|
||||
of BlockType.Block: CodexBlocksKey
|
||||
of BlockType.Both: CodexRepoKey
|
||||
|
||||
let query = Query.init(key, value=false)
|
||||
without queryIter =? (await self.repoDs.query(query)), err:
|
||||
trace "Error querying cids in repo", blockType, err = err.msg
|
||||
return failure(err)
|
||||
|
||||
proc next(): Future[?Cid] {.async.} =
|
||||
await idleAsync()
|
||||
if queryIter.finished:
|
||||
iter.finish
|
||||
else:
|
||||
if pair =? (await queryIter.next()) and cid =? pair.key:
|
||||
doAssert pair.data.len == 0
|
||||
trace "Retrieved record from repo", cid
|
||||
return Cid.init(cid.value).option
|
||||
else:
|
||||
return Cid.none
|
||||
|
||||
iter.next = next
|
||||
return success iter
|
||||
|
||||
proc createBlockExpirationQuery(maxNumber: int, offset: int): ?!Query =
|
||||
let queryKey = ? createBlockExpirationMetadataQueryKey()
|
||||
success Query.init(queryKey, offset = offset, limit = maxNumber)
|
||||
|
||||
method getBlockExpirations*(
|
||||
self: RepoStore,
|
||||
maxNumber: int,
|
||||
offset: int): Future[?!AsyncIter[?BlockExpiration]] {.async, base.} =
|
||||
## Get block expirations from the given RepoStore
|
||||
##
|
||||
|
||||
without query =? createBlockExpirationQuery(maxNumber, offset), err:
|
||||
trace "Unable to format block expirations query"
|
||||
return failure(err)
|
||||
|
||||
without queryIter =? (await self.metaDs.query(query)), err:
|
||||
trace "Unable to execute block expirations query"
|
||||
return failure(err)
|
||||
|
||||
var iter = AsyncIter[?BlockExpiration]()
|
||||
|
||||
proc next(): Future[?BlockExpiration] {.async.} =
|
||||
if not queryIter.finished:
|
||||
if pair =? (await queryIter.next()) and blockKey =? pair.key:
|
||||
let expirationTimestamp = pair.data
|
||||
let cidResult = Cid.init(blockKey.value)
|
||||
if not cidResult.isOk:
|
||||
raiseAssert("Unable to parse CID from blockKey.value: " & blockKey.value & $cidResult.error)
|
||||
return BlockExpiration(
|
||||
cid: cidResult.get,
|
||||
expiration: expirationTimestamp.toSecondsSince1970
|
||||
).some
|
||||
else:
|
||||
discard await queryIter.dispose()
|
||||
iter.finish
|
||||
return BlockExpiration.none
|
||||
|
||||
iter.next = next
|
||||
return success iter
|
||||
|
||||
method close*(self: RepoStore): Future[void] {.async.} =
|
||||
## Close the blockstore, cleaning up resources managed by it.
|
||||
## For some implementations this may be a no-op
|
||||
##
|
||||
|
||||
trace "Closing repostore"
|
||||
|
||||
if not self.metaDs.isNil:
|
||||
(await self.metaDs.close()).expect("Should meta datastore")
|
||||
|
||||
if not self.repoDs.isNil:
|
||||
(await self.repoDs.close()).expect("Should repo datastore")
|
||||
|
||||
proc reserve*(self: RepoStore, bytes: uint): Future[?!void] {.async.} =
|
||||
## Reserve bytes
|
||||
##
|
||||
|
||||
trace "Reserving bytes", reserved = self.quotaReservedBytes, bytes
|
||||
|
||||
if (self.totalUsed + bytes) > self.quotaMaxBytes:
|
||||
trace "Not enough storage quota to reserver", reserve = self.totalUsed + bytes
|
||||
return failure(
|
||||
newException(QuotaNotEnoughError, "Not enough storage quota to reserver"))
|
||||
|
||||
self.quotaReservedBytes += bytes
|
||||
if err =? (await self.metaDs.put(
|
||||
QuotaReservedKey,
|
||||
@(toBytesBE(self.quotaReservedBytes.uint64)))).errorOption:
|
||||
|
||||
trace "Error reserving bytes", err = err.msg
|
||||
|
||||
self.quotaReservedBytes += bytes
|
||||
return failure(err)
|
||||
|
||||
return success()
|
||||
|
||||
proc release*(self: RepoStore, bytes: uint): Future[?!void] {.async.} =
|
||||
## Release bytes
|
||||
##
|
||||
|
||||
trace "Releasing bytes", reserved = self.quotaReservedBytes, bytes
|
||||
|
||||
if (self.quotaReservedBytes.int - bytes.int) < 0:
|
||||
trace "Cannot release this many bytes",
|
||||
quotaReservedBytes = self.quotaReservedBytes, bytes
|
||||
|
||||
return failure("Cannot release this many bytes")
|
||||
|
||||
self.quotaReservedBytes -= bytes
|
||||
if err =? (await self.metaDs.put(
|
||||
QuotaReservedKey,
|
||||
@(toBytesBE(self.quotaReservedBytes.uint64)))).errorOption:
|
||||
|
||||
trace "Error releasing bytes", err = err.msg
|
||||
|
||||
self.quotaReservedBytes -= bytes
|
||||
|
||||
return failure(err)
|
||||
|
||||
trace "Released bytes", bytes
|
||||
self.updateMetrics()
|
||||
return success()
|
||||
|
||||
proc start*(self: RepoStore): Future[void] {.async.} =
|
||||
## Start repo
|
||||
##
|
||||
|
||||
if self.started:
|
||||
trace "Repo already started"
|
||||
return
|
||||
|
||||
trace "Starting repo"
|
||||
|
||||
without total =? await self.metaDs.get(CodexTotalBlocksKey), err:
|
||||
if not (err of DatastoreKeyNotFound):
|
||||
error "Unable to read total number of blocks from metadata store", err = err.msg, key = $CodexTotalBlocksKey
|
||||
|
||||
if total.len > 0:
|
||||
self.totalBlocks = uint64.fromBytesBE(total).uint
|
||||
trace "Number of blocks in store at start", total = self.totalBlocks
|
||||
|
||||
## load current persist and cache bytes from meta ds
|
||||
without quotaUsedBytes =? await self.metaDs.get(QuotaUsedKey), err:
|
||||
if not (err of DatastoreKeyNotFound):
|
||||
error "Error getting cache bytes from datastore",
|
||||
err = err.msg, key = $QuotaUsedKey
|
||||
|
||||
raise newException(Defect, err.msg)
|
||||
|
||||
if quotaUsedBytes.len > 0:
|
||||
self.quotaUsedBytes = uint64.fromBytesBE(quotaUsedBytes).uint
|
||||
|
||||
notice "Current bytes used for cache quota", bytes = self.quotaUsedBytes
|
||||
|
||||
without quotaReservedBytes =? await self.metaDs.get(QuotaReservedKey), err:
|
||||
if not (err of DatastoreKeyNotFound):
|
||||
error "Error getting persist bytes from datastore",
|
||||
err = err.msg, key = $QuotaReservedKey
|
||||
|
||||
raise newException(Defect, err.msg)
|
||||
|
||||
if quotaReservedBytes.len > 0:
|
||||
self.quotaReservedBytes = uint64.fromBytesBE(quotaReservedBytes).uint
|
||||
|
||||
if self.quotaUsedBytes > self.quotaMaxBytes:
|
||||
raiseAssert "All storage quota used, increase storage quota!"
|
||||
|
||||
notice "Current bytes used for persist quota", bytes = self.quotaReservedBytes
|
||||
|
||||
self.updateMetrics()
|
||||
self.started = true
|
||||
|
||||
proc stop*(self: RepoStore): Future[void] {.async.} =
|
||||
## Stop repo
|
||||
##
|
||||
if not self.started:
|
||||
trace "Repo is not started"
|
||||
return
|
||||
|
||||
trace "Stopping repo"
|
||||
await self.close()
|
||||
|
||||
self.started = false
|
||||
|
||||
func new*(
|
||||
T: type RepoStore,
|
||||
repoDs: Datastore,
|
||||
metaDs: Datastore,
|
||||
clock: Clock = SystemClock.new(),
|
||||
postFixLen = 2,
|
||||
quotaMaxBytes = DefaultQuotaBytes,
|
||||
blockTtl = DefaultBlockTtl
|
||||
): RepoStore =
|
||||
## Create new instance of a RepoStore
|
||||
##
|
||||
RepoStore(
|
||||
repoDs: repoDs,
|
||||
metaDs: metaDs,
|
||||
clock: clock,
|
||||
postFixLen: postFixLen,
|
||||
quotaMaxBytes: quotaMaxBytes,
|
||||
blockTtl: blockTtl
|
||||
)
|
||||
export store, types, coders
|
||||
|
||||
47
codex/stores/repostore/coders.nim
Normal file
47
codex/stores/repostore/coders.nim
Normal file
@ -0,0 +1,47 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
##
|
||||
|
||||
import std/sugar
|
||||
import pkg/libp2p/cid
|
||||
import pkg/serde/json
|
||||
import pkg/stew/byteutils
|
||||
import pkg/stew/endians2
|
||||
|
||||
import ./types
|
||||
import ../../errors
|
||||
import ../../merkletree
|
||||
import ../../utils/json
|
||||
|
||||
proc encode*(t: QuotaUsage): seq[byte] = t.toJson().toBytes()
|
||||
proc decode*(T: type QuotaUsage, bytes: seq[byte]): ?!T = T.fromJson(bytes)
|
||||
|
||||
proc encode*(t: BlockMetadata): seq[byte] = t.toJson().toBytes()
|
||||
proc decode*(T: type BlockMetadata, bytes: seq[byte]): ?!T = T.fromJson(bytes)
|
||||
|
||||
proc encode*(t: LeafMetadata): seq[byte] = t.toJson().toBytes()
|
||||
proc decode*(T: type LeafMetadata, bytes: seq[byte]): ?!T = T.fromJson(bytes)
|
||||
|
||||
proc encode*(t: DeleteResult): seq[byte] = t.toJson().toBytes()
|
||||
proc decode*(T: type DeleteResult, bytes: seq[byte]): ?!T = T.fromJson(bytes)
|
||||
|
||||
proc encode*(t: StoreResult): seq[byte] = t.toJson().toBytes()
|
||||
proc decode*(T: type StoreResult, bytes: seq[byte]): ?!T = T.fromJson(bytes)
|
||||
|
||||
proc encode*(i: uint64): seq[byte] =
|
||||
@(i.toBytesBE)
|
||||
|
||||
proc decode*(T: type uint64, bytes: seq[byte]): ?!T =
|
||||
if bytes.len >= sizeof(uint64):
|
||||
success(uint64.fromBytesBE(bytes))
|
||||
else:
|
||||
failure("Not enough bytes to decode `uint64`")
|
||||
|
||||
proc encode*(i: Natural | enum): seq[byte] = cast[uint64](i).encode
|
||||
proc decode*(T: typedesc[Natural | enum], bytes: seq[byte]): ?!T = uint64.decode(bytes).map((ui: uint64) => cast[T](ui))
|
||||
213
codex/stores/repostore/operations.nim
Normal file
213
codex/stores/repostore/operations.nim
Normal file
@ -0,0 +1,213 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/chronos/futures
|
||||
import pkg/datastore
|
||||
import pkg/datastore/typedds
|
||||
import pkg/libp2p/cid
|
||||
import pkg/metrics
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import ./coders
|
||||
import ./types
|
||||
import ../blockstore
|
||||
import ../keyutils
|
||||
import ../../blocktype
|
||||
import ../../clock
|
||||
import ../../logutils
|
||||
import ../../merkletree
|
||||
|
||||
logScope:
|
||||
topics = "codex repostore"
|
||||
|
||||
declareGauge(codex_repostore_blocks, "codex repostore blocks")
|
||||
declareGauge(codex_repostore_bytes_used, "codex repostore bytes used")
|
||||
declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved")
|
||||
|
||||
proc putLeafMetadata*(self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof): Future[?!StoreResultKind] {.async.} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
await self.metaDs.modifyGet(key,
|
||||
proc (maybeCurrMd: ?LeafMetadata): Future[(?LeafMetadata, StoreResultKind)] {.async.} =
|
||||
var
|
||||
md: LeafMetadata
|
||||
res: StoreResultKind
|
||||
|
||||
if currMd =? maybeCurrMd:
|
||||
md = currMd
|
||||
res = AlreadyInStore
|
||||
else:
|
||||
md = LeafMetadata(blkCid: blkCid, proof: proof)
|
||||
res = Stored
|
||||
|
||||
(md.some, res)
|
||||
)
|
||||
|
||||
proc getLeafMetadata*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!LeafMetadata] {.async.} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
without leafMd =? await get[LeafMetadata](self.metaDs, key), err:
|
||||
if err of DatastoreKeyNotFound:
|
||||
return failure(newException(BlockNotFoundError, err.msg))
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
success(leafMd)
|
||||
|
||||
proc updateTotalBlocksCount*(self: RepoStore, plusCount: Natural = 0, minusCount: Natural = 0): Future[?!void] {.async.} =
|
||||
await self.metaDs.modify(CodexTotalBlocksKey,
|
||||
proc (maybeCurrCount: ?Natural): Future[?Natural] {.async.} =
|
||||
let count: Natural =
|
||||
if currCount =? maybeCurrCount:
|
||||
currCount + plusCount - minusCount
|
||||
else:
|
||||
plusCount - minusCount
|
||||
|
||||
self.totalBlocks = count
|
||||
codex_repostore_blocks.set(count.int64)
|
||||
count.some
|
||||
)
|
||||
|
||||
proc updateQuotaUsage*(
|
||||
self: RepoStore,
|
||||
plusUsed: NBytes = 0.NBytes,
|
||||
minusUsed: NBytes = 0.NBytes,
|
||||
plusReserved: NBytes = 0.NBytes,
|
||||
minusReserved: NBytes = 0.NBytes
|
||||
): Future[?!void] {.async.} =
|
||||
await self.metaDs.modify(QuotaUsedKey,
|
||||
proc (maybeCurrUsage: ?QuotaUsage): Future[?QuotaUsage] {.async.} =
|
||||
var usage: QuotaUsage
|
||||
|
||||
if currUsage =? maybeCurrUsage:
|
||||
usage = QuotaUsage(used: currUsage.used + plusUsed - minusUsed, reserved: currUsage.reserved + plusReserved - minusReserved)
|
||||
else:
|
||||
usage = QuotaUsage(used: plusUsed - minusUsed, reserved: plusReserved - minusReserved)
|
||||
|
||||
if usage.used + usage.reserved > self.quotaMaxBytes:
|
||||
raise newException(QuotaNotEnoughError,
|
||||
"Quota usage would exceed the limit. Used: " & $usage.used & ", reserved: " &
|
||||
$usage.reserved & ", limit: " & $self.quotaMaxBytes)
|
||||
else:
|
||||
self.quotaUsage = usage
|
||||
codex_repostore_bytes_used.set(usage.used.int64)
|
||||
codex_repostore_bytes_reserved.set(usage.reserved.int64)
|
||||
return usage.some
|
||||
)
|
||||
|
||||
proc updateBlockMetadata*(
|
||||
self: RepoStore,
|
||||
cid: Cid,
|
||||
plusRefCount: Natural = 0,
|
||||
minusRefCount: Natural = 0,
|
||||
minExpiry: SecondsSince1970 = 0
|
||||
): Future[?!void] {.async.} =
|
||||
if cid.isEmpty:
|
||||
return success()
|
||||
|
||||
without metaKey =? createBlockExpirationMetadataKey(cid), err:
|
||||
return failure(err)
|
||||
|
||||
await self.metaDs.modify(metaKey,
|
||||
proc (maybeCurrBlockMd: ?BlockMetadata): Future[?BlockMetadata] {.async.} =
|
||||
if currBlockMd =? maybeCurrBlockMd:
|
||||
BlockMetadata(
|
||||
size: currBlockMd.size,
|
||||
expiry: max(currBlockMd.expiry, minExpiry),
|
||||
refCount: currBlockMd.refCount + plusRefCount - minusRefCount
|
||||
).some
|
||||
else:
|
||||
raise newException(BlockNotFoundError, "Metadata for block with cid " & $cid & " not found")
|
||||
)
|
||||
|
||||
proc storeBlock*(self: RepoStore, blk: Block, minExpiry: SecondsSince1970): Future[?!StoreResult] {.async.} =
|
||||
if blk.isEmpty:
|
||||
return success(StoreResult(kind: AlreadyInStore))
|
||||
|
||||
without metaKey =? createBlockExpirationMetadataKey(blk.cid), err:
|
||||
return failure(err)
|
||||
|
||||
without blkKey =? makePrefixKey(self.postFixLen, blk.cid), err:
|
||||
return failure(err)
|
||||
|
||||
await self.metaDs.modifyGet(metaKey,
|
||||
proc (maybeCurrMd: ?BlockMetadata): Future[(?BlockMetadata, StoreResult)] {.async.} =
|
||||
var
|
||||
md: BlockMetadata
|
||||
res: StoreResult
|
||||
|
||||
if currMd =? maybeCurrMd:
|
||||
if currMd.size == blk.data.len.NBytes:
|
||||
md = BlockMetadata(size: currMd.size, expiry: max(currMd.expiry, minExpiry), refCount: currMd.refCount)
|
||||
res = StoreResult(kind: AlreadyInStore)
|
||||
|
||||
# making sure that the block acutally is stored in the repoDs
|
||||
without hasBlock =? await self.repoDs.has(blkKey), err:
|
||||
raise err
|
||||
|
||||
if not hasBlock:
|
||||
warn "Block metadata is present, but block is absent. Restoring block.", cid = blk.cid
|
||||
if err =? (await self.repoDs.put(blkKey, blk.data)).errorOption:
|
||||
raise err
|
||||
else:
|
||||
raise newException(CatchableError, "Repo already stores a block with the same cid but with a different size, cid: " & $blk.cid)
|
||||
else:
|
||||
md = BlockMetadata(size: blk.data.len.NBytes, expiry: minExpiry, refCount: 0)
|
||||
res = StoreResult(kind: Stored, used: blk.data.len.NBytes)
|
||||
if err =? (await self.repoDs.put(blkKey, blk.data)).errorOption:
|
||||
raise err
|
||||
|
||||
(md.some, res)
|
||||
)
|
||||
|
||||
proc tryDeleteBlock*(self: RepoStore, cid: Cid, expiryLimit = SecondsSince1970.low): Future[?!DeleteResult] {.async.} =
|
||||
if cid.isEmpty:
|
||||
return success(DeleteResult(kind: InUse))
|
||||
|
||||
without metaKey =? createBlockExpirationMetadataKey(cid), err:
|
||||
return failure(err)
|
||||
|
||||
without blkKey =? makePrefixKey(self.postFixLen, cid), err:
|
||||
return failure(err)
|
||||
|
||||
await self.metaDs.modifyGet(metaKey,
|
||||
proc (maybeCurrMd: ?BlockMetadata): Future[(?BlockMetadata, DeleteResult)] {.async.} =
|
||||
var
|
||||
maybeMeta: ?BlockMetadata
|
||||
res: DeleteResult
|
||||
|
||||
if currMd =? maybeCurrMd:
|
||||
if currMd.refCount == 0 or currMd.expiry < expiryLimit:
|
||||
maybeMeta = BlockMetadata.none
|
||||
res = DeleteResult(kind: Deleted, released: currMd.size)
|
||||
|
||||
if err =? (await self.repoDs.delete(blkKey)).errorOption:
|
||||
raise err
|
||||
else:
|
||||
maybeMeta = currMd.some
|
||||
res = DeleteResult(kind: InUse)
|
||||
else:
|
||||
maybeMeta = BlockMetadata.none
|
||||
res = DeleteResult(kind: NotFound)
|
||||
|
||||
# making sure that the block acutally is removed from the repoDs
|
||||
without hasBlock =? await self.repoDs.has(blkKey), err:
|
||||
raise err
|
||||
|
||||
if hasBlock:
|
||||
warn "Block metadata is absent, but block is present. Removing block.", cid
|
||||
if err =? (await self.repoDs.delete(blkKey)).errorOption:
|
||||
raise err
|
||||
|
||||
(maybeMeta, res)
|
||||
)
|
||||
398
codex/stores/repostore/store.nim
Normal file
398
codex/stores/repostore/store.nim
Normal file
@ -0,0 +1,398 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/chronos/futures
|
||||
import pkg/datastore
|
||||
import pkg/datastore/typedds
|
||||
import pkg/libp2p/[cid, multicodec]
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import ./coders
|
||||
import ./types
|
||||
import ./operations
|
||||
import ../blockstore
|
||||
import ../keyutils
|
||||
import ../queryiterhelper
|
||||
import ../../blocktype
|
||||
import ../../clock
|
||||
import ../../logutils
|
||||
import ../../merkletree
|
||||
import ../../utils
|
||||
|
||||
export blocktype, cid
|
||||
|
||||
logScope:
|
||||
topics = "codex repostore"
|
||||
|
||||
###########################################################
|
||||
# BlockStore API
|
||||
###########################################################
|
||||
|
||||
method getBlock*(self: RepoStore, cid: Cid): Future[?!Block] {.async.} =
|
||||
## Get a block from the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
if cid.isEmpty:
|
||||
trace "Empty block, ignoring"
|
||||
return cid.emptyBlock
|
||||
|
||||
without key =? makePrefixKey(self.postFixLen, cid), err:
|
||||
trace "Error getting key from provider", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without data =? await self.repoDs.get(key), err:
|
||||
if not (err of DatastoreKeyNotFound):
|
||||
trace "Error getting block from datastore", err = err.msg, key
|
||||
return failure(err)
|
||||
|
||||
return failure(newException(BlockNotFoundError, err.msg))
|
||||
|
||||
trace "Got block for cid", cid
|
||||
return Block.new(cid, data, verify = true)
|
||||
|
||||
method getBlockAndProof*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.async.} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
without blk =? await self.getBlock(leafMd.blkCid), err:
|
||||
return failure(err)
|
||||
|
||||
success((blk, leafMd.proof))
|
||||
|
||||
method getBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!Block] {.async.} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
await self.getBlock(leafMd.blkCid)
|
||||
|
||||
method getBlock*(self: RepoStore, address: BlockAddress): Future[?!Block] =
|
||||
## Get a block from the blockstore
|
||||
##
|
||||
|
||||
if address.leaf:
|
||||
self.getBlock(address.treeCid, address.index)
|
||||
else:
|
||||
self.getBlock(address.cid)
|
||||
|
||||
method ensureExpiry*(
|
||||
self: RepoStore,
|
||||
cid: Cid,
|
||||
expiry: SecondsSince1970
|
||||
): Future[?!void] {.async.} =
|
||||
## Ensure that block's associated expiry is at least given timestamp
|
||||
## If the current expiry is lower then it is updated to the given one, otherwise it is left intact
|
||||
##
|
||||
|
||||
if expiry <= 0:
|
||||
return failure(newException(ValueError, "Expiry timestamp must be larger then zero"))
|
||||
|
||||
await self.updateBlockMetadata(cid, minExpiry = expiry)
|
||||
|
||||
method ensureExpiry*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
expiry: SecondsSince1970
|
||||
): Future[?!void] {.async.} =
|
||||
## Ensure that block's associated expiry is at least given timestamp
|
||||
## If the current expiry is lower then it is updated to the given one, otherwise it is left intact
|
||||
##
|
||||
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
await self.ensureExpiry(leafMd.blkCid, expiry)
|
||||
|
||||
method putCidAndProof*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blkCid: Cid,
|
||||
proof: CodexProof
|
||||
): Future[?!void] {.async.} =
|
||||
## Put a block to the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
treeCid = treeCid
|
||||
index = index
|
||||
blkCid = blkCid
|
||||
|
||||
trace "Storing LeafMetadata"
|
||||
|
||||
without res =? await self.putLeafMetadata(treeCid, index, blkCid, proof), err:
|
||||
return failure(err)
|
||||
|
||||
if blkCid.mcodec == BlockCodec:
|
||||
if res == Stored:
|
||||
if err =? (await self.updateBlockMetadata(blkCid, plusRefCount = 1)).errorOption:
|
||||
return failure(err)
|
||||
trace "Leaf metadata stored, block refCount incremented"
|
||||
else:
|
||||
trace "Leaf metadata already exists"
|
||||
|
||||
return success()
|
||||
|
||||
method getCidAndProof*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async.} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
success((leafMd.blkCid, leafMd.proof))
|
||||
|
||||
method getCid*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural
|
||||
): Future[?!Cid] {.async.} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
success(leafMd.blkCid)
|
||||
|
||||
method putBlock*(
|
||||
self: RepoStore,
|
||||
blk: Block,
|
||||
ttl = Duration.none): Future[?!void] {.async.} =
|
||||
## Put a block to the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = blk.cid
|
||||
|
||||
let expiry = self.clock.now() + (ttl |? self.blockTtl).seconds
|
||||
|
||||
without res =? await self.storeBlock(blk, expiry), err:
|
||||
return failure(err)
|
||||
|
||||
if res.kind == Stored:
|
||||
trace "Block Stored"
|
||||
if err =? (await self.updateQuotaUsage(plusUsed = res.used)).errorOption:
|
||||
# rollback changes
|
||||
without delRes =? await self.tryDeleteBlock(blk.cid), err:
|
||||
return failure(err)
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.updateTotalBlocksCount(plusCount = 1)).errorOption:
|
||||
return failure(err)
|
||||
|
||||
if onBlock =? self.onBlockStored:
|
||||
await onBlock(blk.cid)
|
||||
else:
|
||||
trace "Block already exists"
|
||||
|
||||
return success()
|
||||
|
||||
method delBlock*(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
|
||||
## Delete a block from the blockstore when block refCount is 0 or block is expired
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
trace "Attempting to delete a block"
|
||||
|
||||
without res =? await self.tryDeleteBlock(cid, self.clock.now()), err:
|
||||
return failure(err)
|
||||
|
||||
if res.kind == Deleted:
|
||||
trace "Block deleted"
|
||||
if err =? (await self.updateTotalBlocksCount(minusCount = 1)).errorOption:
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.updateQuotaUsage(minusUsed = res.released)).errorOption:
|
||||
return failure(err)
|
||||
elif res.kind == InUse:
|
||||
trace "Block in use, refCount > 0 and not expired"
|
||||
else:
|
||||
trace "Block not found in store"
|
||||
|
||||
return success()
|
||||
|
||||
method delBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!void] {.async.} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
if err of BlockNotFoundError:
|
||||
return success()
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.updateBlockMetadata(leafMd.blkCid, minusRefCount = 1)).errorOption:
|
||||
if not (err of BlockNotFoundError):
|
||||
return failure(err)
|
||||
|
||||
await self.delBlock(leafMd.blkCid) # safe delete, only if refCount == 0
|
||||
|
||||
method hasBlock*(self: RepoStore, cid: Cid): Future[?!bool] {.async.} =
|
||||
## Check if the block exists in the blockstore
|
||||
##
|
||||
|
||||
logScope:
|
||||
cid = cid
|
||||
|
||||
if cid.isEmpty:
|
||||
trace "Empty block, ignoring"
|
||||
return success true
|
||||
|
||||
without key =? makePrefixKey(self.postFixLen, cid), err:
|
||||
trace "Error getting key from provider", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
return await self.repoDs.has(key)
|
||||
|
||||
method hasBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!bool] {.async.} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
if err of BlockNotFoundError:
|
||||
return success(false)
|
||||
else:
|
||||
return failure(err)
|
||||
|
||||
await self.hasBlock(leafMd.blkCid)
|
||||
|
||||
method listBlocks*(
|
||||
self: RepoStore,
|
||||
blockType = BlockType.Manifest
|
||||
): Future[?!AsyncIter[?Cid]] {.async.} =
|
||||
## Get the list of blocks in the RepoStore.
|
||||
## This is an intensive operation
|
||||
##
|
||||
|
||||
var
|
||||
iter = AsyncIter[?Cid]()
|
||||
|
||||
let key =
|
||||
case blockType:
|
||||
of BlockType.Manifest: CodexManifestKey
|
||||
of BlockType.Block: CodexBlocksKey
|
||||
of BlockType.Both: CodexRepoKey
|
||||
|
||||
let query = Query.init(key, value=false)
|
||||
without queryIter =? (await self.repoDs.query(query)), err:
|
||||
trace "Error querying cids in repo", blockType, err = err.msg
|
||||
return failure(err)
|
||||
|
||||
proc next(): Future[?Cid] {.async.} =
|
||||
await idleAsync()
|
||||
if queryIter.finished:
|
||||
iter.finish
|
||||
else:
|
||||
if pair =? (await queryIter.next()) and cid =? pair.key:
|
||||
doAssert pair.data.len == 0
|
||||
trace "Retrieved record from repo", cid
|
||||
return Cid.init(cid.value).option
|
||||
else:
|
||||
return Cid.none
|
||||
|
||||
iter.next = next
|
||||
return success iter
|
||||
|
||||
proc createBlockExpirationQuery(maxNumber: int, offset: int): ?!Query =
|
||||
let queryKey = ? createBlockExpirationMetadataQueryKey()
|
||||
success Query.init(queryKey, offset = offset, limit = maxNumber)
|
||||
|
||||
method getBlockExpirations*(
|
||||
self: RepoStore,
|
||||
maxNumber: int,
|
||||
offset: int): Future[?!AsyncIter[BlockExpiration]] {.async, base.} =
|
||||
## Get iterator with block expirations
|
||||
##
|
||||
|
||||
without beQuery =? createBlockExpirationQuery(maxNumber, offset), err:
|
||||
error "Unable to format block expirations query", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without queryIter =? await query[BlockMetadata](self.metaDs, beQuery), err:
|
||||
error "Unable to execute block expirations query", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without asyncQueryIter =? await queryIter.toAsyncIter(), err:
|
||||
error "Unable to convert QueryIter to AsyncIter", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
let
|
||||
filteredIter = await asyncQueryIter.filterSuccess()
|
||||
blockExpIter = await mapFilter[KeyVal[BlockMetadata], BlockExpiration](filteredIter,
|
||||
proc (kv: KeyVal[BlockMetadata]): Future[?BlockExpiration] {.async.} =
|
||||
without cid =? Cid.init(kv.key.value).mapFailure, err:
|
||||
error "Failed decoding cid", err = err.msg
|
||||
return BlockExpiration.none
|
||||
|
||||
BlockExpiration(cid: cid, expiry: kv.value.expiry).some
|
||||
)
|
||||
|
||||
success(blockExpIter)
|
||||
|
||||
method close*(self: RepoStore): Future[void] {.async.} =
|
||||
## Close the blockstore, cleaning up resources managed by it.
|
||||
## For some implementations this may be a no-op
|
||||
##
|
||||
|
||||
trace "Closing repostore"
|
||||
|
||||
if not self.metaDs.isNil:
|
||||
(await self.metaDs.close()).expect("Should meta datastore")
|
||||
|
||||
if not self.repoDs.isNil:
|
||||
(await self.repoDs.close()).expect("Should repo datastore")
|
||||
|
||||
###########################################################
|
||||
# RepoStore procs
|
||||
###########################################################
|
||||
|
||||
proc reserve*(self: RepoStore, bytes: NBytes): Future[?!void] {.async.} =
|
||||
## Reserve bytes
|
||||
##
|
||||
|
||||
trace "Reserving bytes", bytes
|
||||
|
||||
await self.updateQuotaUsage(plusReserved = bytes)
|
||||
|
||||
proc release*(self: RepoStore, bytes: NBytes): Future[?!void] {.async.} =
|
||||
## Release bytes
|
||||
##
|
||||
|
||||
trace "Releasing bytes", bytes
|
||||
|
||||
await self.updateQuotaUsage(minusReserved = bytes)
|
||||
|
||||
proc start*(self: RepoStore): Future[void] {.async.} =
|
||||
## Start repo
|
||||
##
|
||||
|
||||
if self.started:
|
||||
trace "Repo already started"
|
||||
return
|
||||
|
||||
trace "Starting rep"
|
||||
if err =? (await self.updateTotalBlocksCount()).errorOption:
|
||||
raise newException(CodexError, err.msg)
|
||||
|
||||
if err =? (await self.updateQuotaUsage()).errorOption:
|
||||
raise newException(CodexError, err.msg)
|
||||
|
||||
self.started = true
|
||||
|
||||
proc stop*(self: RepoStore): Future[void] {.async.} =
|
||||
## Stop repo
|
||||
##
|
||||
if not self.started:
|
||||
trace "Repo is not started"
|
||||
return
|
||||
|
||||
trace "Stopping repo"
|
||||
await self.close()
|
||||
|
||||
self.started = false
|
||||
109
codex/stores/repostore/types.nim
Normal file
109
codex/stores/repostore/types.nim
Normal file
@ -0,0 +1,109 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/datastore
|
||||
import pkg/datastore/typedds
|
||||
import pkg/libp2p/cid
|
||||
import pkg/questionable
|
||||
|
||||
import ../blockstore
|
||||
import ../../clock
|
||||
import ../../errors
|
||||
import ../../merkletree
|
||||
import ../../systemclock
|
||||
import ../../units
|
||||
|
||||
const
|
||||
DefaultBlockTtl* = 24.hours
|
||||
DefaultQuotaBytes* = 8.GiBs
|
||||
|
||||
type
|
||||
QuotaNotEnoughError* = object of CodexError
|
||||
|
||||
RepoStore* = ref object of BlockStore
|
||||
postFixLen*: int
|
||||
repoDs*: Datastore
|
||||
metaDs*: TypedDatastore
|
||||
clock*: Clock
|
||||
quotaMaxBytes*: NBytes
|
||||
quotaUsage*: QuotaUsage
|
||||
totalBlocks*: Natural
|
||||
blockTtl*: Duration
|
||||
started*: bool
|
||||
|
||||
QuotaUsage* {.serialize.} = object
|
||||
used*: NBytes
|
||||
reserved*: NBytes
|
||||
|
||||
BlockMetadata* {.serialize.} = object
|
||||
expiry*: SecondsSince1970
|
||||
size*: NBytes
|
||||
refCount*: Natural
|
||||
|
||||
LeafMetadata* {.serialize.} = object
|
||||
blkCid*: Cid
|
||||
proof*: CodexProof
|
||||
|
||||
BlockExpiration* {.serialize.} = object
|
||||
cid*: Cid
|
||||
expiry*: SecondsSince1970
|
||||
|
||||
DeleteResultKind* {.serialize.} = enum
|
||||
Deleted = 0, # block removed from store
|
||||
InUse = 1, # block not removed, refCount > 0 and not expired
|
||||
NotFound = 2 # block not found in store
|
||||
|
||||
DeleteResult* {.serialize.} = object
|
||||
kind*: DeleteResultKind
|
||||
released*: NBytes
|
||||
|
||||
StoreResultKind* {.serialize.} = enum
|
||||
Stored = 0, # new block stored
|
||||
AlreadyInStore = 1 # block already in store
|
||||
|
||||
StoreResult* {.serialize.} = object
|
||||
kind*: StoreResultKind
|
||||
used*: NBytes
|
||||
|
||||
func quotaUsedBytes*(self: RepoStore): NBytes =
|
||||
self.quotaUsage.used
|
||||
|
||||
func quotaReservedBytes*(self: RepoStore): NBytes =
|
||||
self.quotaUsage.reserved
|
||||
|
||||
func totalUsed*(self: RepoStore): NBytes =
|
||||
(self.quotaUsedBytes + self.quotaReservedBytes)
|
||||
|
||||
func available*(self: RepoStore): NBytes =
|
||||
return self.quotaMaxBytes - self.totalUsed
|
||||
|
||||
func available*(self: RepoStore, bytes: NBytes): bool =
|
||||
return bytes < self.available()
|
||||
|
||||
func new*(
|
||||
T: type RepoStore,
|
||||
repoDs: Datastore,
|
||||
metaDs: Datastore,
|
||||
clock: Clock = SystemClock.new(),
|
||||
postFixLen = 2,
|
||||
quotaMaxBytes = DefaultQuotaBytes,
|
||||
blockTtl = DefaultBlockTtl
|
||||
): RepoStore =
|
||||
## Create new instance of a RepoStore
|
||||
##
|
||||
RepoStore(
|
||||
repoDs: repoDs,
|
||||
metaDs: TypedDatastore.init(metaDs),
|
||||
clock: clock,
|
||||
postFixLen: postFixLen,
|
||||
quotaMaxBytes: quotaMaxBytes,
|
||||
blockTtl: blockTtl,
|
||||
onBlockStored: CidCallback.none
|
||||
)
|
||||
@ -38,7 +38,6 @@ type
|
||||
StoreStream* = ref object of SeekableStream
|
||||
store*: BlockStore # Store where to lookup block contents
|
||||
manifest*: Manifest # List of block CIDs
|
||||
pad*: bool # Pad last block to manifest.blockSize?
|
||||
|
||||
method initStream*(s: StoreStream) =
|
||||
if s.objName.len == 0:
|
||||
@ -57,13 +56,15 @@ proc new*(
|
||||
result = StoreStream(
|
||||
store: store,
|
||||
manifest: manifest,
|
||||
pad: pad,
|
||||
offset: 0)
|
||||
|
||||
result.initStream()
|
||||
|
||||
method `size`*(self: StoreStream): int =
|
||||
bytes(self.manifest, self.pad).int
|
||||
## The size of a StoreStream is the size of the original dataset, without
|
||||
## padding or parity blocks.
|
||||
let m = self.manifest
|
||||
(if m.protected: m.originalDatasetSize else: m.datasetSize).int
|
||||
|
||||
proc `size=`*(self: StoreStream, size: int)
|
||||
{.error: "Setting the size is forbidden".} =
|
||||
|
||||
@ -46,9 +46,13 @@ proc `'nb`*(n: string): NBytes = parseInt(n).NBytes
|
||||
logutils.formatIt(NBytes): $it
|
||||
|
||||
const
|
||||
MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz
|
||||
KiB = 1024.NBytes # ByteSz, 1 kibibyte = 1,024 ByteSz
|
||||
MiB = KiB * 1024 # ByteSz, 1 mebibyte = 1,048,576 ByteSz
|
||||
GiB = MiB * 1024 # ByteSz, 1 gibibyte = 1,073,741,824 ByteSz
|
||||
|
||||
proc KiBs*(v: Natural): NBytes = v.NBytes * KiB
|
||||
proc MiBs*(v: Natural): NBytes = v.NBytes * MiB
|
||||
proc GiBs*(v: Natural): NBytes = v.NBytes * GiB
|
||||
|
||||
func divUp*[T: NBytes](a, b : T): int =
|
||||
## Division with result rounded up (rather than truncated as in 'div')
|
||||
|
||||
@ -1,35 +1,38 @@
|
||||
import std/sets
|
||||
import std/sequtils
|
||||
import pkg/chronos
|
||||
import pkg/questionable/results
|
||||
|
||||
import ./validationconfig
|
||||
import ./market
|
||||
import ./clock
|
||||
import ./logutils
|
||||
|
||||
export market
|
||||
export sets
|
||||
export validationconfig
|
||||
|
||||
type
|
||||
Validation* = ref object
|
||||
slots: HashSet[SlotId]
|
||||
maxSlots: int
|
||||
clock: Clock
|
||||
market: Market
|
||||
subscriptions: seq[Subscription]
|
||||
running: Future[void]
|
||||
periodicity: Periodicity
|
||||
proofTimeout: UInt256
|
||||
config: ValidationConfig
|
||||
|
||||
logScope:
|
||||
topics = "codex validator"
|
||||
|
||||
proc new*(
|
||||
_: type Validation,
|
||||
clock: Clock,
|
||||
market: Market,
|
||||
maxSlots: int
|
||||
_: type Validation,
|
||||
clock: Clock,
|
||||
market: Market,
|
||||
config: ValidationConfig
|
||||
): Validation =
|
||||
## Create a new Validation instance
|
||||
Validation(clock: clock, market: market, maxSlots: maxSlots)
|
||||
Validation(clock: clock, market: market, config: config)
|
||||
|
||||
proc slots*(validation: Validation): seq[SlotId] =
|
||||
validation.slots.toSeq
|
||||
@ -43,13 +46,29 @@ proc waitUntilNextPeriod(validation: Validation) {.async.} =
|
||||
trace "Waiting until next period", currentPeriod = period
|
||||
await validation.clock.waitUntil(periodEnd.truncate(int64) + 1)
|
||||
|
||||
func groupIndexForSlotId*(slotId: SlotId,
|
||||
validationGroups: ValidationGroups): uint16 =
|
||||
let slotIdUInt256 = UInt256.fromBytesBE(slotId.toArray)
|
||||
(slotIdUInt256 mod validationGroups.u256).truncate(uint16)
|
||||
|
||||
func maxSlotsConstraintRespected(validation: Validation): bool =
|
||||
validation.config.maxSlots == 0 or
|
||||
validation.slots.len < validation.config.maxSlots
|
||||
|
||||
func shouldValidateSlot(validation: Validation, slotId: SlotId): bool =
|
||||
if (validationGroups =? validation.config.groups):
|
||||
(groupIndexForSlotId(slotId, validationGroups) ==
|
||||
validation.config.groupIndex) and
|
||||
validation.maxSlotsConstraintRespected
|
||||
else:
|
||||
validation.maxSlotsConstraintRespected
|
||||
|
||||
proc subscribeSlotFilled(validation: Validation) {.async.} =
|
||||
proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) =
|
||||
let slotId = slotId(requestId, slotIndex)
|
||||
if slotId notin validation.slots:
|
||||
if validation.slots.len < validation.maxSlots:
|
||||
trace "Adding slot", slotId
|
||||
validation.slots.incl(slotId)
|
||||
if validation.shouldValidateSlot(slotId):
|
||||
trace "Adding slot", slotId
|
||||
validation.slots.incl(slotId)
|
||||
let subscription = await validation.market.subscribeSlotFilled(onSlotFilled)
|
||||
validation.subscriptions.add(subscription)
|
||||
|
||||
|
||||
36
codex/validationconfig.nim
Normal file
36
codex/validationconfig.nim
Normal file
@ -0,0 +1,36 @@
|
||||
import std/strformat
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
type
|
||||
ValidationGroups* = range[2..65535]
|
||||
MaxSlots* = int
|
||||
ValidationConfig* = object
|
||||
maxSlots: MaxSlots
|
||||
groups: ?ValidationGroups
|
||||
groupIndex: uint16
|
||||
|
||||
func init*(
|
||||
_: type ValidationConfig,
|
||||
maxSlots: MaxSlots,
|
||||
groups: ?ValidationGroups,
|
||||
groupIndex: uint16 = 0): ?!ValidationConfig =
|
||||
if maxSlots < 0:
|
||||
return failure "The value of maxSlots must be greater than " &
|
||||
fmt"or equal to 0! (got: {maxSlots})"
|
||||
if validationGroups =? groups and groupIndex >= uint16(validationGroups):
|
||||
return failure "The value of the group index must be less than " &
|
||||
fmt"validation groups! (got: {groupIndex = }, " &
|
||||
fmt"groups = {validationGroups})"
|
||||
|
||||
success ValidationConfig(
|
||||
maxSlots: maxSlots, groups: groups, groupIndex: groupIndex)
|
||||
|
||||
func maxSlots*(config: ValidationConfig): MaxSlots =
|
||||
config.maxSlots
|
||||
|
||||
func groups*(config: ValidationConfig): ?ValidationGroups =
|
||||
config.groups
|
||||
|
||||
func groupIndex*(config: ValidationConfig): uint16 =
|
||||
config.groupIndex
|
||||
@ -121,6 +121,9 @@ switch("define", "ctt_asm=false")
|
||||
# Allow the use of old-style case objects for nim config compatibility
|
||||
switch("define", "nimOldCaseObjects")
|
||||
|
||||
# Enable compat mode for Chronos V4
|
||||
switch("define", "chronosHandleException")
|
||||
|
||||
# begin Nimble config (version 1)
|
||||
when system.fileExists("nimble.paths"):
|
||||
include "nimble.paths"
|
||||
|
||||
@ -24,9 +24,9 @@ RUN echo "export PATH=$PATH:$HOME/.cargo/bin" >> $BASH_ENV
|
||||
|
||||
WORKDIR ${BUILD_HOME}
|
||||
COPY . .
|
||||
RUN make clean
|
||||
RUN make -j ${MAKE_PARALLEL} update
|
||||
RUN make -j ${MAKE_PARALLEL}
|
||||
RUN make -j ${MAKE_PARALLEL} cirdl
|
||||
|
||||
# Create
|
||||
FROM ${IMAGE}
|
||||
@ -35,10 +35,10 @@ ARG APP_HOME
|
||||
ARG NAT_IP_AUTO
|
||||
|
||||
WORKDIR ${APP_HOME}
|
||||
COPY --from=builder ${BUILD_HOME}/build/codex /usr/local/bin
|
||||
COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
|
||||
COPY --from=builder ${BUILD_HOME}/openapi.yaml .
|
||||
COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
|
||||
RUN apt-get update && apt-get install -y libgomp1 bash curl jq && rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
|
||||
ENV NAT_IP_AUTO=${NAT_IP_AUTO}
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["codex"]
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Environment variables from files
|
||||
# If set to file path, read the file and export the variables
|
||||
# If set to directory path, read all files in the directory and export the variables
|
||||
if [[ -n "${ENV_PATH}" ]]; then
|
||||
set -a
|
||||
[[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done
|
||||
@ -50,6 +52,34 @@ if [ -n "${PRIV_KEY}" ]; then
|
||||
echo "Private key set"
|
||||
fi
|
||||
|
||||
# Circuit downloader
|
||||
# cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress]
|
||||
if [[ "$@" == *"prover"* ]]; then
|
||||
echo "Prover is enabled - Run Circuit downloader"
|
||||
|
||||
# Set variables required by cirdl from command line arguments when passed
|
||||
for arg in data-dir circuit-dir eth-provider marketplace-address; do
|
||||
arg_value=$(grep -o "${arg}=[^ ,]\+" <<< $@ | awk -F '=' '{print $2}')
|
||||
if [[ -n "${arg_value}" ]]; then
|
||||
var_name=$(tr '[:lower:]' '[:upper:]' <<< "CODEX_${arg//-/_}")
|
||||
export "${var_name}"="${arg_value}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Set circuit dir from CODEX_CIRCUIT_DIR variables if set
|
||||
if [[ -z "${CODEX_CIRCUIT_DIR}" ]]; then
|
||||
export CODEX_CIRCUIT_DIR="${CODEX_DATA_DIR}/circuits"
|
||||
fi
|
||||
|
||||
# Download circuit
|
||||
mkdir -p "${CODEX_CIRCUIT_DIR}"
|
||||
chmod 700 "${CODEX_CIRCUIT_DIR}"
|
||||
download="cirdl ${CODEX_CIRCUIT_DIR} ${CODEX_ETH_PROVIDER} ${CODEX_MARKETPLACE_ADDRESS}"
|
||||
echo "${download}"
|
||||
eval "${download}"
|
||||
[[ $? -ne 0 ]] && { echo "Failed to download circuit files"; exit 1; }
|
||||
fi
|
||||
|
||||
# Run
|
||||
echo "Run Codex node"
|
||||
exec "$@"
|
||||
|
||||
@ -1,68 +0,0 @@
|
||||
# Download Flow
|
||||
Sequence of interactions that result in dat blocks being transferred across the network.
|
||||
|
||||
## Local Store
|
||||
When data is available in the local blockstore,
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
actor Alice
|
||||
participant API
|
||||
Alice->>API: Download(CID)
|
||||
API->>+Node/StoreStream: Retrieve(CID)
|
||||
loop Get manifest block, then data blocks
|
||||
Node/StoreStream->>NetworkStore: GetBlock(CID)
|
||||
NetworkStore->>LocalStore: GetBlock(CID)
|
||||
LocalStore->>NetworkStore: Block
|
||||
NetworkStore->>Node/StoreStream: Block
|
||||
end
|
||||
Node/StoreStream->>Node/StoreStream: Handle erasure coding
|
||||
Node/StoreStream->>-API: Data stream
|
||||
API->>Alice: Stream download of block
|
||||
```
|
||||
|
||||
## Network Store
|
||||
When data is not found ih the local blockstore, the block-exchange engine is used to discover the location of the block within the network. Connection will be established to the node(s) that have the block, and exchange can take place.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
box
|
||||
actor Alice
|
||||
participant API
|
||||
participant Node/StoreStream
|
||||
participant NetworkStore
|
||||
participant Discovery
|
||||
participant Engine
|
||||
end
|
||||
box
|
||||
participant OtherNode
|
||||
end
|
||||
Alice->>API: Download(CID)
|
||||
API->>+Node/StoreStream: Retrieve(CID)
|
||||
Node/StoreStream->>-API: Data stream
|
||||
API->>Alice: Download stream begins
|
||||
loop Get manifest block, then data blocks
|
||||
Node/StoreStream->>NetworkStore: GetBlock(CID)
|
||||
NetworkStore->>Engine: RequestBlock(CID)
|
||||
opt CID not known
|
||||
Engine->>Discovery: Discovery Block
|
||||
Discovery->>Discovery: Locates peers who provide block
|
||||
Discovery->>Engine: Peers
|
||||
Engine->>Engine: Update peers admin
|
||||
end
|
||||
Engine->>Engine: Select optimal peer
|
||||
Engine->>OtherNode: Send WantHave list
|
||||
OtherNode->>Engine: Send BlockPresence
|
||||
Engine->>Engine: Update peers admin
|
||||
Engine->>Engine: Decide to buy block
|
||||
Engine->>OtherNode: Send WantBlock list
|
||||
OtherNode->>Engine: Send Block
|
||||
Engine->>NetworkStore: Block
|
||||
NetworkStore->>NetworkStore: Add to Local store
|
||||
NetworkStore->>Node/StoreStream: Resolve Block
|
||||
Node/StoreStream->>Node/StoreStream: Handle erasure coding
|
||||
Node/StoreStream->>API: Push data to stream
|
||||
end
|
||||
API->>Alice: Download stream finishes
|
||||
```
|
||||
|
||||
@ -1,444 +0,0 @@
|
||||
# Running a Local Codex Network with Marketplace Support
|
||||
|
||||
This tutorial will teach you how to run a small Codex network with the _storage marketplace_ enabled; i.e., the functionality in Codex which allows participants to offer and buy storage in a market, ensuring that storage providers honor their part of the deal by means of cryptographic proofs.
|
||||
|
||||
To complete this tutorial, you will need:
|
||||
|
||||
* the [geth](https://github.com/ethereum/go-ethereum) Ethereum client;
|
||||
* a Codex binary, which [you can compile from source](https://github.com/codex-storage/nim-codex?tab=readme-ov-file#build-and-run).
|
||||
|
||||
We will also be using [bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) syntax throughout. If you use a different shell, you may need to adapt things to your platform.
|
||||
|
||||
In this tutorial, you will:
|
||||
|
||||
1. [Set Up a Geth PoA network](#1-set-up-a-geth-poa-network);
|
||||
2. [Set up The Marketplace](#2-set-up-the-marketplace);
|
||||
3. [Run Codex](#3-run-codex);
|
||||
4. [Buy and Sell Storage in the Marketplace](#4-buy-and-sell-storage-on-the-marketplace).
|
||||
|
||||
We strongly suggest you to create a folder (e.g. `marketplace-tutorial`), and switch into it before beginning.
|
||||
|
||||
## 1. Set Up a Geth PoA Network
|
||||
|
||||
For this tutorial, we will use a simple [Proof-of-Authority](https://github.com/ethereum/EIPs/issues/225) network with geth. The first step is creating a _signer account_: an account which will be used by geth to sign the blocks in the network. Any block signed by a signer is accepted as valid.
|
||||
|
||||
### 1.1. Create a Signer Account
|
||||
|
||||
To create a signer account, run:
|
||||
|
||||
```bash
|
||||
geth account new --datadir geth-data
|
||||
```
|
||||
|
||||
The account generator will ask you to input a password, which you can leave blank. It will then print some information, including the account's public address:
|
||||
|
||||
```bash
|
||||
INFO [03-22|12:58:05.637] Maximum peer count ETH=50 total=50
|
||||
INFO [03-22|12:58:05.638] Smartcard socket not found, disabling err="stat /run/pcscd/pcscd.comm: no such file or directory"
|
||||
Your new account is locked with a password. Please give a password. Do not forget this password.
|
||||
Password:
|
||||
Repeat password:
|
||||
|
||||
Your new key was generated
|
||||
|
||||
Public address of the key: 0x93976895c4939d99837C8e0E1779787718EF8368
|
||||
...
|
||||
```
|
||||
|
||||
In this example, the public address of the signer account is `0x93976895c4939d99837C8e0E1779787718EF8368`. Yours will print a different address. Save it for later usage.
|
||||
|
||||
Next set an environment variable for later usage:
|
||||
|
||||
```sh
|
||||
export GETH_SIGNER_ADDR="0x0000000000000000000000000000000000000000"
|
||||
echo ${GETH_SIGNER_ADDR} > geth_signer_address.txt
|
||||
```
|
||||
|
||||
### 1.2. Configure The Network and Create the Genesis Block
|
||||
|
||||
The next step is telling geth what kind of network you want to run. We will be running a [pre-merge](https://ethereum.org/en/roadmap/merge/) network with Proof-of-Authority consensus. To get that working, create a `network.json` file.
|
||||
|
||||
If you set the GETH_SIGNER_ADDR variable above you can run to create the `network.json` file:
|
||||
|
||||
```sh
|
||||
echo "{\"config\": { \"chainId\": 12345, \"homesteadBlock\": 0, \"eip150Block\": 0, \"eip155Block\": 0, \"eip158Block\": 0, \"byzantiumBlock\": 0, \"constantinopleBlock\": 0, \"petersburgBlock\": 0, \"istanbulBlock\": 0, \"berlinBlock\": 0, \"londonBlock\": 0, \"arrowGlacierBlock\": 0, \"grayGlacierBlock\": 0, \"clique\": { \"period\": 1, \"epoch\": 30000 } }, \"difficulty\": \"1\", \"gasLimit\": \"8000000\", \"extradata\": \"0x0000000000000000000000000000000000000000000000000000000000000000${GETH_SIGNER_ADDR:2}0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\", \"alloc\": { \"${GETH_SIGNER_ADDR}\": { \"balance\": \"10000000000000000000000\"}}}" > network.json
|
||||
```
|
||||
|
||||
You can also manually create the file with the following content modified with your signer private key:
|
||||
|
||||
```json
|
||||
{
|
||||
"config": {
|
||||
"chainId": 12345,
|
||||
"homesteadBlock": 0,
|
||||
"eip150Block": 0,
|
||||
"eip155Block": 0,
|
||||
"eip158Block": 0,
|
||||
"byzantiumBlock": 0,
|
||||
"constantinopleBlock": 0,
|
||||
"petersburgBlock": 0,
|
||||
"istanbulBlock": 0,
|
||||
"berlinBlock": 0,
|
||||
"londonBlock": 0,
|
||||
"arrowGlacierBlock": 0,
|
||||
"grayGlacierBlock": 0,
|
||||
"clique": {
|
||||
"period": 1,
|
||||
"epoch": 30000
|
||||
}
|
||||
},
|
||||
"difficulty": "1",
|
||||
"gasLimit": "8000000",
|
||||
"extradata": "0x000000000000000000000000000000000000000000000000000000000000000093976895c4939d99837C8e0E1779787718EF83680000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
|
||||
"alloc": {
|
||||
"0x93976895c4939d99837C8e0E1779787718EF8368": {
|
||||
"balance": "10000000000000000000000"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note that the signer account address is embedded in two different places:
|
||||
* inside of the `"extradata"` string, surrounded by zeroes and stripped of its `0x` prefix;
|
||||
* as an entry key in the `alloc` session.
|
||||
Make sure to replace that ID with the account ID that you wrote down in Step 1.1.
|
||||
|
||||
|
||||
Once `network.json` is created, you can initialize the network with:
|
||||
|
||||
```bash
|
||||
geth init --datadir geth-data network.json
|
||||
```
|
||||
|
||||
### 1.3. Start your PoA Node
|
||||
|
||||
We are now ready to start our $1$-node, private blockchain. To launch the signer node, open a separate terminal on the same working directory and run:
|
||||
|
||||
```bash
|
||||
geth\
|
||||
--datadir geth-data\
|
||||
--networkid 12345\
|
||||
--unlock ${GETH_SIGNER_ADDR}\
|
||||
--nat extip:127.0.0.1\
|
||||
--netrestrict 127.0.0.0/24\
|
||||
--mine\
|
||||
--miner.etherbase ${GETH_SIGNER_ADDR}\
|
||||
--http\
|
||||
--allow-insecure-unlock
|
||||
```
|
||||
|
||||
Note that, once again, the signer account created in Step 1.1 appears both in `--unlock` and `--allow-insecure-unlock`. Make sure you have the `GETH_SIGNER_ADDR` set.
|
||||
|
||||
Geth will prompt you to insert the account's password as it starts up. Once you do that, it should be able to start up and begin "mining" blocks.
|
||||
|
||||
## 2. Set Up The Marketplace
|
||||
|
||||
You will need to open new terminal for this section and geth needs to be running already. Setting up the Codex marketplace entails:
|
||||
|
||||
1. Deploying the Codex Marketplace contracts to our private blockchain
|
||||
2. Setup Ethereum accounts we will use to buy and sell storage in the Codex marketplace
|
||||
3. Provisioning those accounts with the required token balances
|
||||
|
||||
### 2.1. Deploy the Codex Marketplace Contracts
|
||||
|
||||
To deploy the contracts, start by cloning the Codex contracts repository locally and installing its dependencies:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/codex-storage/codex-contracts-eth
|
||||
cd codex-contracts-eth
|
||||
npm install
|
||||
```
|
||||
You now must **wait until $256$ blocks are mined in your PoA network**, or deploy will fail. This should take about $4$ minutes and $30$ seconds. You can check which block height you are currently at by running:
|
||||
|
||||
```bash
|
||||
geth attach --exec web3.eth.blockNumber ../geth-data/geth.ipc
|
||||
```
|
||||
|
||||
once that gets past $256$, you are ready to go. To deploy contracts, run:
|
||||
|
||||
```bash
|
||||
export DISTTEST_NETWORK_URL=http://localhost:8545 # bootstrap node
|
||||
npx hardhat --network codexdisttestnetwork deploy && cd ../
|
||||
```
|
||||
|
||||
If the command completes successfully, you are ready to prepare the accounts.
|
||||
|
||||
### 2.2. Generate the Required Accounts
|
||||
|
||||
We will run $2$ Codex nodes: a **storage provider**, which will sell storage on the network, and a **client**, which will buy and use such storage; we therefore need two valid Ethereum accounts. We could create random accounts by using one of the many tools available to that end but, since this is a tutorial running on a local private network, we will simply provide you with two pre-made accounts along with their private keys which you can copy and paste instead:
|
||||
|
||||
First make sure you're back in the `marketplace-tutorial` folder and not the `codex-contracts-eth` subfolder. Then set these variables:
|
||||
|
||||
**Storage:**
|
||||
```sh
|
||||
export ETH_STORAGE_ADDR=0x45BC5ca0fbdD9F920Edd12B90908448C30F32a37
|
||||
export ETH_STORAGE_PK=0x06c7ac11d4ee1d0ccb53811b71802fa92d40a5a174afad9f2cb44f93498322c3
|
||||
echo $ETH_STORAGE_PK > storage.pkey && chmod 0600 storage.pkey
|
||||
```
|
||||
|
||||
**Client:**
|
||||
```sh
|
||||
export ETH_CLIENT_ADDR=0x9F0C62Fe60b22301751d6cDe1175526b9280b965
|
||||
export ETH_CLIENT_PK=0x5538ec03c956cb9d0bee02a25b600b0225f1347da4071d0fd70c521fdc63c2fc
|
||||
echo $ETH_CLIENT_PK > client.pkey && chmod 0600 client.pkey
|
||||
```
|
||||
|
||||
### 2.3. Provision Accounts with Tokens
|
||||
|
||||
We now need to transfer some ETH to each of the accounts, as well as provide them with some Codex tokens for the storage node to use as collateral and for the client node to buy actual storage.
|
||||
|
||||
Although the process is not particularly complicated, I suggest you use [the script we prepared](https://github.com/gmega/local-codex-bare/blob/main/scripts/mint-tokens.js) for that. This script, essentially:
|
||||
|
||||
1. reads the Marketplace contract address and its ABI from the deployment data;
|
||||
2. transfers $1$ ETH from the signer account to a target account if the target account has no ETH balance;
|
||||
3. mints $n$ Codex tokens and adds it into the target account's balance.
|
||||
|
||||
To use the script, just download it into a local file named `mint-tokens.js`, for instance using curl:
|
||||
|
||||
```bash
|
||||
# set the contract file location
|
||||
export CONTRACT_DEPLOY_FULL="codex-contracts-eth/deployments/codexdisttestnetwork"
|
||||
export GETH_SIGNER_ADDR=$(cat geth_signer_address.txt)
|
||||
# download script
|
||||
curl https://raw.githubusercontent.com/gmega/codex-local-bare/main/scripts/mint-tokens.js -o mint-tokens.js
|
||||
```
|
||||
|
||||
```bash
|
||||
# Installs Web3-js
|
||||
npm install web3
|
||||
# Provides tokens to the storage account.
|
||||
node ./mint-tokens.js $CONTRACT_DEPLOY_FULL/TestToken.json $GETH_SIGNER_ADDR 0x45BC5ca0fbdD9F920Edd12B90908448C30F32a37 10000000000
|
||||
# Provides tokens to the client account.
|
||||
node ./mint-tokens.js $CONTRACT_DEPLOY_FULL/TestToken.json $GETH_SIGNER_ADDR 0x9F0C62Fe60b22301751d6cDe1175526b9280b965 10000000000
|
||||
```
|
||||
|
||||
If you get a message like `Usage: mint-tokens.js <token-hardhat-deploy-json> <signer-account> <receiver-account> <token-ammount>` then you need to ensure you have
|
||||
|
||||
## 3. Run Codex
|
||||
|
||||
With accounts and geth in place, we can now start the Codex nodes.
|
||||
|
||||
### 3.1. Storage Node
|
||||
|
||||
The storage node will be the one storing data and submitting the proofs of storage to the chain. To do that, it needs access to:
|
||||
|
||||
1. the address of the Marketplace contract that has been deployed to the local geth node in [Step 2.1](#21-deploy-the-codex-marketplace-contracts);
|
||||
2. the sample ceremony files which are shipped in the Codex contracts repo.
|
||||
|
||||
Recall you have clone the `codex-contracts-eth` repository in Step 2.1. All of the required files are in there.
|
||||
|
||||
**Address of the Marketplace Contract.** The contract address can be found inside of the file `codex-contracts-eth/deployments/codexdisttestnetwork/Marketplace.json`:
|
||||
|
||||
```bash
|
||||
grep '"address":' ${CONTRACT_DEPLOY_FULL}/Marketplace.json
|
||||
```
|
||||
|
||||
which should print something like:
|
||||
```sh
|
||||
"address": "0x8891732D890f5A7B7181fBc70F7482DE28a7B60f",
|
||||
```
|
||||
|
||||
Then run the following with the correct market place address:
|
||||
```sh
|
||||
export MARKETPLACE_ADDRESS="0x0000000000000000000000000000000000000000"
|
||||
echo ${MARKETPLACE_ADDRESS} > marketplace_address.txt
|
||||
```
|
||||
|
||||
**Prover ceremony files.** The ceremony files are under the `codex-contracts-eth/verifier/networks/codexdisttestnetwork` subdirectory. There are three of them: `proof_main.r1cs`, `proof_main.zkey`, and `prooof_main.wasm`. We will need all of them to start the Codex storage node.
|
||||
|
||||
**Starting the storage node.** Let:
|
||||
|
||||
* `PROVER_ASSETS` contain the directory where the prover ceremony files are located. **This must be an absolute path**;
|
||||
* `CODEX_BINARY` contain the location of your Codex binary;
|
||||
* `MARKETPLACE_ADDRESS` contain the address of the Marketplace contract (obtained above).
|
||||
|
||||
Set these paths into environment variables (modify it with the correct paths if you changed them above):
|
||||
|
||||
```sh
|
||||
export CONTRACT_DEPLOY_FULL=$(realpath "codex-contracts-eth/deployments/codexdisttestnetwork")
|
||||
export PROVER_ASSETS=$(realpath "codex-contracts-eth/verifier/networks/codexdisttestnetwork/")
|
||||
export CODEX_BINARY=$(realpath "../build/codex")
|
||||
export MARKETPLACE_ADDRESS=$(cat marketplace_address.txt)
|
||||
```
|
||||
|
||||
To launch the storage node, run:
|
||||
|
||||
```bash
|
||||
${CODEX_BINARY}\
|
||||
--data-dir=./codex-storage\
|
||||
--listen-addrs=/ip4/0.0.0.0/tcp/8080\
|
||||
--api-port=8000\
|
||||
--disc-port=8090\
|
||||
persistence\
|
||||
--eth-provider=http://localhost:8545\
|
||||
--eth-private-key=./storage.pkey\
|
||||
--marketplace-address=${MARKETPLACE_ADDRESS}\
|
||||
--validator\
|
||||
--validator-max-slots=1000\
|
||||
prover\
|
||||
--circom-r1cs=${PROVER_ASSETS}/proof_main.r1cs\
|
||||
--circom-wasm=${PROVER_ASSETS}/proof_main.wasm\
|
||||
--circom-zkey=${PROVER_ASSETS}/proof_main.zkey
|
||||
```
|
||||
|
||||
**Starting the client node.**
|
||||
|
||||
The client node is started similarly except that:
|
||||
|
||||
* we need to pass the SPR of the storage node so it can form a network with it;
|
||||
* since it does not run any proofs, it does not require any ceremony files.
|
||||
|
||||
We get the Signed Peer Record (SPR) of the storage node so we can bootstrap the client node with it. To get the SPR, issue the following call:
|
||||
|
||||
```bash
|
||||
curl -H 'Accept: text/plain' 'http://localhost:8000/api/codex/v1/spr'
|
||||
```
|
||||
|
||||
You should get the SPR back starting with `spr:`. Next set these paths into environment variables:
|
||||
|
||||
```bash
|
||||
# set the SPR for the storage node
|
||||
export STORAGE_NODE_SPR=$(curl -H 'Accept: text/plain' 'http://localhost:8000/api/codex/v1/spr')
|
||||
# basic vars
|
||||
export CONTRACT_DEPLOY_FULL=$(realpath "codex-contracts-eth/deployments/codexdisttestnetwork")
|
||||
export PROVER_ASSETS=$(realpath "codex-contracts-eth/verifier/networks/codexdisttestnetwork/")
|
||||
export CODEX_BINARY=$(realpath "../build/codex")
|
||||
export MARKETPLACE_ADDRESS=$(cat marketplace_address.txt)
|
||||
```
|
||||
|
||||
```bash
|
||||
${CODEX_BINARY}\
|
||||
--data-dir=./codex-client\
|
||||
--listen-addrs=/ip4/0.0.0.0/tcp/8081\
|
||||
--api-port=8001\
|
||||
--disc-port=8091\
|
||||
--bootstrap-node=${STORAGE_NODE_SPR}\
|
||||
persistence\
|
||||
--eth-provider=http://localhost:8545\
|
||||
--eth-private-key=./client.pkey\
|
||||
--marketplace-address=${MARKETPLACE_ADDRESS}
|
||||
```
|
||||
|
||||
## 4. Buy and Sell Storage on the Marketplace
|
||||
|
||||
Any storage negotiation has two sides: a buyer and a seller. Before we can actually request storage, therefore, we must first put some of it for sale.
|
||||
|
||||
### 4.1 Sell Storage
|
||||
|
||||
The following request will cause the storage node to put out $50\text{MB}$ of storage for sale for $1$ hour, at a price of $1$ Codex token per byte per second, while expressing that it's willing to take at most a $1000$ Codex token penalty for not fulfilling its part of the contract.[^1]
|
||||
|
||||
```bash
|
||||
curl 'http://localhost:8000/api/codex/v1/sales/availability' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"totalSize": "50000000",
|
||||
"duration": "3600",
|
||||
"minPrice": "1",
|
||||
"maxCollateral": "1000"
|
||||
}'
|
||||
```
|
||||
|
||||
This should return a response with an id a string (e.g. `"id": "0x552ef12a2ee64ca22b237335c7e1df884df36d22bfd6506b356936bc718565d4"`) which identifies this storage offer. To check the current storage offers for this node, you can issue:
|
||||
|
||||
```bash
|
||||
curl 'http://localhost:8000/api/codex/v1/sales/availability'
|
||||
```
|
||||
|
||||
This should print a list of offers, with the one you just created figuring among them.
|
||||
|
||||
## 4.2. Buy Storage
|
||||
|
||||
Before we can buy storage, we must have some actual data to request storage for. Start by uploading a small file to your client node. On Linux you could, for instance, use `dd` to generate a $100KB$ file:
|
||||
|
||||
```bash
|
||||
dd if=/dev/urandom of=./data.bin bs=100K count=1
|
||||
```
|
||||
|
||||
but any small file will do. Assuming your file is named `data.bin`, you can upload it with:
|
||||
|
||||
```bash
|
||||
curl "http://localhost:8001/api/codex/v1/data" --data-bin @data.bin
|
||||
```
|
||||
|
||||
Once the upload completes, you should see a CID (e.g. `zDvZRwzm2mK7tvDzKScRLapqGdgNTLyyEBvx1TQY37J2CdWdS6Sj`) for the file printed to the terminal. Use that CID in the purchase request:
|
||||
|
||||
```bash
|
||||
export CID=zDvZRwzm2mK7tvDzKScRLapqGdgNTLyyEBvx1TQY37J2CdWdS6Sj
|
||||
export EXPIRY_TIME=$((1000 + $(date +%s))) # current time + 1000 seconds
|
||||
# adjust expiry_time as desired, see below
|
||||
```
|
||||
|
||||
```bash
|
||||
curl "http://localhost:8001/api/codex/v1/storage/request/${CID}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "{
|
||||
\"duration\": \"1200\",
|
||||
\"reward\": \"1\",
|
||||
\"proofProbability\": \"3\",
|
||||
\"expiry\": \"${EXPIRY_TIME}\",
|
||||
\"nodes\": 1,
|
||||
\"tolerance\": 0,
|
||||
\"collateral\": \"1000\"
|
||||
}"
|
||||
```
|
||||
|
||||
The parameters under `--data` say that:
|
||||
|
||||
1. we want to purchase storage for our file for $20$ minutes (`"duration": "1200"`);
|
||||
2. we are willing to pay up to $1$ token per byte, per second (`"reward": "1"`);
|
||||
3. our file will be split into four pieces (`"nodes": 3` and `"tolerance": 1`), so that we only need three pieces to rebuild the file; i.e., we can tolerate that at most one node stops storing our data; either due to failure or other reasons;
|
||||
4. we demand `1000` tokens in collateral from storage providers for each piece. Since there are $4$ such pieces, there will be `4000` in total collateral committed by all of the storage providers taken together once our request is fulfilled.
|
||||
|
||||
Finally, the `expiry` puts a cap on the block time at which our request expires. This has to be at most `current block time + duration`, which means this request can fail if you input the wrong number, which you likely will if you do not know what the current block time is. Fear not, however, as you can try an an arbitrary number (e.g. `1000`), and look at the failure message:
|
||||
|
||||
`Expiry needs to be in future. Now: 1711995463`
|
||||
|
||||
to compute a valid one. Just take the number in the error message and add the duration; i.e., `1711995463 + 1200 = 1711996663`, then use the resulting number (`1711996663`) as expiry and things should work. The request should return a purchase ID (e.g. `1d0ec5261e3364f8b9d1cf70324d70af21a9b5dccba380b24eb68b4762249185`), which you can use track the completion of your request in the marketplace.
|
||||
|
||||
## 4.3. Track your Storage Requests
|
||||
|
||||
POSTing a storage request will make it available in the storage market, and a storage node will eventually pick it up.
|
||||
|
||||
You can poll the status of your request by means of:
|
||||
```bash
|
||||
export STORAGE_PURCHASE_ID="1d0ec5261e3364f8b9d1cf70324d70af21a9b5dccba380b24eb68b4762249185"
|
||||
curl "http://localhost:8001/api/codex/v1/storage/purchases/${STORAGE_PURCHASE_ID}"
|
||||
```
|
||||
|
||||
For instance:
|
||||
|
||||
```bash
|
||||
> curl 'http://localhost:8001/api/codex/v1/storage/purchases/6c698cd0ad71c41982f83097d6fa75beb582924e08a658357a1cd4d7a2a6766d'
|
||||
```
|
||||
|
||||
This returns a result like:
|
||||
|
||||
```json
|
||||
{
|
||||
"requestId": "0x6c698cd0ad71c41982f83097d6fa75beb582924e08a658357a1cd4d7a2a6766d",
|
||||
"request": {
|
||||
"client": "0xed6c3c20358f0217919a30c98d72e29ceffedc33",
|
||||
"ask": {
|
||||
"slots": 3,
|
||||
"slotSize": "262144",
|
||||
"duration": "1000",
|
||||
"proofProbability": "3",
|
||||
"reward": "1",
|
||||
"collateral": "1",
|
||||
"maxSlotLoss": 1
|
||||
},
|
||||
"content": {
|
||||
"cid": "zDvZRwzm3nnkekFLCACmWyKdkYixsX3j9gJhkvFtfYA5K9bpXQnC"
|
||||
},
|
||||
"expiry": "1711992852",
|
||||
"nonce": "0x9f5e651ecd3bf73c914f8ed0b1088869c64095c0d7bd50a38fc92ebf66ff5915",
|
||||
"id": "0x6c698cd0ad71c41982f83097d6fa75beb582924e08a658357a1cd4d7a2a6766d"
|
||||
},
|
||||
"state": "submitted",
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
Shows that a request has been submitted but has not yet been filled. Your request will be successful once `"state"` shows `"started"`. Anything other than that means the request has not been completely processed yet, and an `"error"` state other than `null` means it failed.
|
||||
|
||||
[^1]: Codex files get partitioned into pieces called "slots" and distributed to various storage providers. The collateral refers to one such slot, and will be slowly eaten away as the storage provider fails to deliver timely proofs, but the actual logic is [more involved than that](https://github.com/codex-storage/codex-contracts-eth/blob/6c9f797f408608958714024b9055fcc330e3842f/contracts/Marketplace.sol#L209).
|
||||
@ -1,176 +0,0 @@
|
||||
# Codex Two-Client Test
|
||||
|
||||
The two-client test is a manual test you can perform to check your setup and familiarize yourself with the Codex API. These steps will guide you through running and connecting two nodes, in order to upload a file to one and then download that file from the other. This test also includes running a local blockchain node in order to have the Marketplace functionality available. However, running a local blockchain node is not strictly necessary, and you can skip steps marked as optional if you choose not start a local blockchain node.
|
||||
|
||||
## Prerequisite
|
||||
|
||||
Make sure you have built the client, and can run it as explained in the [README](../README.md).
|
||||
|
||||
## Steps
|
||||
|
||||
### 0. Setup blockchain node (optional)
|
||||
|
||||
You need to have installed NodeJS and npm in order to spinup a local blockchain node.
|
||||
|
||||
Go to directory `vendor/codex-contracts-eth` and run these two commands:
|
||||
```
|
||||
npm ci
|
||||
npm start
|
||||
```
|
||||
|
||||
This will launch a local Ganache blockchain.
|
||||
|
||||
### 1. Launch Node #1
|
||||
|
||||
Open a terminal and run:
|
||||
- Mac/Unx: `"build/codex" --data-dir="$(pwd)/Data1" --listen-addrs="/ip4/127.0.0.1/tcp/8070" --api-port=8080 --disc-port=8090`
|
||||
- Windows: `"build/codex.exe" --data-dir="Data1" --listen-addrs="/ip4/127.0.0.1/tcp/8070" --api-port=8080 --disc-port=8090`
|
||||
|
||||
Optionally, if you want to use the Marketplace blockchain functionality, you need to also include these flags: `--persistence --eth-account=<account>`, where `account` can be one following:
|
||||
|
||||
- `0x70997970C51812dc3A010C7d01b50e0d17dc79C8`
|
||||
- `0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC`
|
||||
- `0x90F79bf6EB2c4f870365E785982E1f101E93b906`
|
||||
- `0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65`
|
||||
|
||||
**For each node use a different account!**
|
||||
|
||||
| Argument | Description |
|
||||
|----------------|-----------------------------------------------------------------------|
|
||||
| `data-dir` | We specify a relative path where the node will store its data. |
|
||||
| `listen-addrs` | Multiaddress where the node will accept connections from other nodes. |
|
||||
| `api-port` | Port on localhost where the node will expose its API. |
|
||||
| `disc-port` | Port the node will use for its discovery service. |
|
||||
| `persistence` | Enables Marketplace functionality. Requires a blockchain connection. |
|
||||
| `eth-account` | Defines which blockchain account the node should use. |
|
||||
|
||||
Codex uses sane defaults for most of its arguments. Here we specify some explicitly for the purpose of this walk-through.
|
||||
|
||||
### 2. Sign of life
|
||||
|
||||
Run the command :
|
||||
|
||||
```bash
|
||||
curl -X GET http://127.0.0.1:8080/api/codex/v1/debug/info
|
||||
```
|
||||
|
||||
This GET request will return the node's debug information. The response will be in JSON and should look like:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "16Uiu2HAmJ3TSfPnrJNedHy2DMsjTqwBiVAQQqPo579DuMgGxmG99",
|
||||
"addrs": [
|
||||
"/ip4/127.0.0.1/tcp/8070"
|
||||
],
|
||||
"repo": "/Users/user/projects/nim-codex/Data1",
|
||||
"spr": "spr:CiUIAhIhA1AL2J7EWfg7x77iOrR9YYBisY6CDtU2nEhuwDaQyjpkEgIDARo8CicAJQgCEiEDUAvYnsRZ-DvHvuI6tH1hgGKxjoIO1TacSG7ANpDKOmQQ2MWasAYaCwoJBH8AAAGRAh-aKkYwRAIgB2ooPfAyzWEJDe8hD2OXKOBnyTOPakc4GzqKqjM2OGoCICraQLPWf0oSEuvmSroFebVQx-3SDtMqDoIyWhjq1XFF",
|
||||
"announceAddresses": [
|
||||
"/ip4/127.0.0.1/tcp/8070"
|
||||
],
|
||||
"table": {
|
||||
"localNode": {
|
||||
"nodeId": "f6e6d48fa7cd171688249a57de0c1aba15e88308c07538c91e1310c9f48c860a",
|
||||
"peerId": "16Uiu2HAmJ3TSfPnrJNedHy2DMsjTqwBiVAQQqPo579DuMgGxmG99",
|
||||
"record": "...",
|
||||
"address": "0.0.0.0:8090",
|
||||
"seen": false
|
||||
},
|
||||
"nodes": []
|
||||
},
|
||||
"codex": {
|
||||
"version": "untagged build",
|
||||
"revision": "b3e626a5"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Description |
|
||||
| ------- | ---------------------------------------------------------------------------------------- |
|
||||
| `id` | Id of the node. Also referred to as 'peerId'. |
|
||||
| `addrs` | Multiaddresses currently open to accept connections from other nodes. |
|
||||
| `repo` | Path of this node's data folder. |
|
||||
| `spr` | Signed Peer Record, encoded information about this node and its location in the network. |
|
||||
| `announceAddresses` | Multiaddresses used for annoucning this node
|
||||
| `table` | Table of nodes present in the node's DHT
|
||||
| `codex` | Codex version information
|
||||
|
||||
### 3. Launch Node #2
|
||||
|
||||
We will need the signed peer record (SPR) from the first node that you got in the previous step.
|
||||
|
||||
Replace `<SPR HERE>` in the following command with the SPR returned from the previous command. (Note that it should include the `spr:` at the beginning.)
|
||||
|
||||
Open a new terminal and run:
|
||||
- Mac/Linux: `"build/codex" --data-dir="$(pwd)/Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=<SPR HERE>`
|
||||
- Windows: `"build/codex.exe" --data-dir="Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=<SPR HERE>`
|
||||
|
||||
Alternatively on Mac, Linux, or MSYS2 and a recent Codex binary you can run it in one command like:
|
||||
|
||||
```sh
|
||||
"build/codex" --data-dir="$(pwd)/Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=$(curl -H "Accept: text/plain" http://127.0.0.1:8080/api/codex/v1/spr)
|
||||
```
|
||||
|
||||
Notice we're using a new data-dir, and we've increased each port number by one. This is needed so that the new node won't try to open ports already in use by the first node.
|
||||
|
||||
We're now also including the `bootstrap-node` argument. This allows us to link the new node to another one, bootstrapping our own little peer-to-peer network. (SPR strings always start with "spr:".)
|
||||
|
||||
### 4. Connect The Two
|
||||
|
||||
Normally the two nodes will automatically connect. If they do not automatically connect or you want to manually connect nodes you can use the peerId to connect nodes.
|
||||
|
||||
You can get the first node's peer id by running the following command and finding the `"peerId"` in the results:
|
||||
|
||||
```bash
|
||||
curl -X GET -H "Accept: text/plain" http://127.0.0.1:8081/api/codex/v1/debug/info
|
||||
```
|
||||
|
||||
Next replace `<PEER ID HERE>` in the following command with the peerId returned from the previous command:
|
||||
|
||||
```bash
|
||||
curl -X GET http://127.0.0.1:8080/api/codex/v1/connect/<PEER ID HERE>?addrs=/ip4/127.0.0.1/tcp/8071
|
||||
```
|
||||
|
||||
Alternatively on Mac, Linux, or MSYS2 and a recent Codex binary you can run it in one command like:
|
||||
|
||||
```bash
|
||||
curl -X GET http://127.0.0.1:8080/api/codex/v1/connect/$(curl -X GET -H "Accept: text/plain" http://127.0.0.1:8081/api/codex/v1/peerid)\?addrs=/ip4/127.0.0.1/tcp/8071
|
||||
```
|
||||
|
||||
Notice that we are sending the peerId and the multiaddress of node 2 to the `/connect` endpoint of node 1. This provides node 1 all the information it needs to communicate with node 2. The response to this request should be `Successfully connected to peer`.
|
||||
|
||||
### 5. Upload The File
|
||||
|
||||
We're now ready to upload a file to the network. In this example we'll use node 1 for uploading and node 2 for downloading. But the reverse also works.
|
||||
|
||||
Next replace `<FILE PATH>` with the path to the file you want to upload in the following command:
|
||||
|
||||
```bash
|
||||
curl -H "Content-Type: application/octet-stream" -H "Expect: 100-continue" -T "<FILE PATH>" 127.0.0.1:8080/api/codex/v1/data -X POST
|
||||
```
|
||||
|
||||
(Hint: if curl is reluctant to show you the response, add `-o <FILENAME>` to write the result to a file.)
|
||||
|
||||
Depending on the file size this may take a moment. Codex is processing the file by cutting it into blocks and generating erasure-recovery data. When the process is finished, the request will return the content-identifier (CID) of the uploaded file. It should look something like `zdj7WVxH8HHHenKtid8Vkgv5Z5eSUbCxxr8xguTUBMCBD8F2S`.
|
||||
|
||||
### 6. Download The File
|
||||
|
||||
Replace `<CID>` with the identifier returned in the previous step. Replace `<OUTPUT FILE>` with the filename where you want to store the downloaded file.
|
||||
|
||||
```bash
|
||||
curl 127.0.0.1:8081/api/codex/v1/data/<CID>/network --output <OUTPUT FILE>
|
||||
```
|
||||
|
||||
Notice we are connecting to the second node in order to download the file. The CID we provide contains the information needed to locate the file within the network.
|
||||
|
||||
### 7. Verify The Results
|
||||
|
||||
If your file is downloaded and identical to the file you uploaded, then this manual test has passed. Rejoice! If on the other hand that didn't happen or you were unable to complete any of these steps, please leave us a message detailing your troubles.
|
||||
|
||||
## Notes
|
||||
|
||||
When using the Ganache blockchain, there are some deviations from the expected behavior, mainly linked to how blocks are mined, which affects certain functionalities in the Sales module.
|
||||
Therefore, if you are manually testing processes such as payout collection after a request is finished or proof submissions, you need to mine some blocks manually for it to work correctly. You can do this by using the following curl command:
|
||||
|
||||
```bash
|
||||
$ curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"evm_mine","params":[],"id":67}' 127.0.0.1:8545
|
||||
```
|
||||
102
openapi.yaml
102
openapi.yaml
@ -23,6 +23,8 @@ components:
|
||||
Id:
|
||||
type: string
|
||||
description: 32bits identifier encoded in hex-decimal string.
|
||||
minLength: 66
|
||||
maxLength: 66
|
||||
example: 0x...
|
||||
|
||||
BigInt:
|
||||
@ -81,33 +83,12 @@ components:
|
||||
id:
|
||||
$ref: "#/components/schemas/PeerId"
|
||||
|
||||
ErasureParameters:
|
||||
type: object
|
||||
properties:
|
||||
totalChunks:
|
||||
type: integer
|
||||
|
||||
PoRParameters:
|
||||
description: Parameters for Proof of Retrievability
|
||||
type: object
|
||||
properties:
|
||||
u:
|
||||
type: string
|
||||
publicKey:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
|
||||
Content:
|
||||
type: object
|
||||
description: Parameters specifying the content
|
||||
properties:
|
||||
cid:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
erasure:
|
||||
$ref: "#/components/schemas/ErasureParameters"
|
||||
por:
|
||||
$ref: "#/components/schemas/PoRParameters"
|
||||
|
||||
DebugInfo:
|
||||
type: object
|
||||
@ -136,7 +117,7 @@ components:
|
||||
$ref: "#/components/schemas/Duration"
|
||||
minPrice:
|
||||
type: string
|
||||
description: Minimum price to be paid (in amount of tokens) as decimal string
|
||||
description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
|
||||
maxCollateral:
|
||||
type: string
|
||||
description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
|
||||
@ -168,7 +149,39 @@ components:
|
||||
$ref: "#/components/schemas/StorageRequest"
|
||||
slotIndex:
|
||||
type: string
|
||||
description: Slot Index as hexadecimal string
|
||||
description: Slot Index as decimal string
|
||||
|
||||
SlotAgent:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
$ref: "#/components/schemas/SlotId"
|
||||
slotIndex:
|
||||
type: string
|
||||
description: Slot Index as decimal string
|
||||
requestId:
|
||||
$ref: "#/components/schemas/Id"
|
||||
request:
|
||||
$ref: "#/components/schemas/StorageRequest"
|
||||
reservation:
|
||||
$ref: "#/components/schemas/Reservation"
|
||||
state:
|
||||
type: string
|
||||
description: Description of the slot's
|
||||
enum:
|
||||
- SaleCancelled
|
||||
- SaleDownloading
|
||||
- SaleErrored
|
||||
- SaleFailed
|
||||
- SaleFilled
|
||||
- SaleFilling
|
||||
- SaleFinished
|
||||
- SaleIgnored
|
||||
- SaleInitialProving
|
||||
- SalePayout
|
||||
- SalePreparing
|
||||
- SaleProving
|
||||
- SaleUnknown
|
||||
|
||||
Reservation:
|
||||
type: object
|
||||
@ -183,7 +196,7 @@ components:
|
||||
$ref: "#/components/schemas/Id"
|
||||
slotIndex:
|
||||
type: string
|
||||
description: Slot Index as hexadecimal string
|
||||
description: Slot Index as decimal string
|
||||
|
||||
StorageRequestCreation:
|
||||
type: object
|
||||
@ -259,6 +272,15 @@ components:
|
||||
state:
|
||||
type: string
|
||||
description: Description of the Request's state
|
||||
enum:
|
||||
- cancelled
|
||||
- error
|
||||
- failed
|
||||
- finished
|
||||
- pending
|
||||
- started
|
||||
- submitted
|
||||
- unknown
|
||||
error:
|
||||
type: string
|
||||
description: If Request failed, then here is presented the error message
|
||||
@ -308,15 +330,15 @@ components:
|
||||
quotaMaxBytes:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Maximum storage space used by the node"
|
||||
description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
|
||||
quotaUsedBytes:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Amount of storage space currently in use"
|
||||
description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
|
||||
quotaReservedBytes:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Amount of storage space reserved"
|
||||
description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
|
||||
|
||||
servers:
|
||||
- url: "http://localhost:8080/api/codex/v1"
|
||||
@ -491,7 +513,7 @@ paths:
|
||||
$ref: "#/components/schemas/Slot"
|
||||
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/sales/slots/{slotId}":
|
||||
get:
|
||||
@ -511,7 +533,7 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Slot"
|
||||
$ref: "#/components/schemas/SlotAgent"
|
||||
|
||||
"400":
|
||||
description: Invalid or missing SlotId
|
||||
@ -520,13 +542,13 @@ paths:
|
||||
description: Host is not in an active sale for the slot
|
||||
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/sales/availability":
|
||||
get:
|
||||
summary: "Returns storage that is for sale"
|
||||
tags: [ Marketplace ]
|
||||
operationId: getOfferedStorage
|
||||
operationId: getAvailabilities
|
||||
responses:
|
||||
"200":
|
||||
description: Retrieved storage availabilities of the node
|
||||
@ -535,11 +557,11 @@ paths:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/SalesAvailability"
|
||||
$ref: "#/components/schemas/SalesAvailabilityREAD"
|
||||
"500":
|
||||
description: Error getting unused availabilities
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
post:
|
||||
summary: "Offers storage for sale"
|
||||
@ -564,7 +586,7 @@ paths:
|
||||
"500":
|
||||
description: Error reserving availability
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
"/sales/availability/{id}":
|
||||
patch:
|
||||
summary: "Updates availability"
|
||||
@ -597,10 +619,10 @@ paths:
|
||||
"500":
|
||||
description: Error reserving availability
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/sales/availability/{id}/reservations":
|
||||
patch:
|
||||
get:
|
||||
summary: "Get availability's reservations"
|
||||
description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
|
||||
operationId: getReservations
|
||||
@ -628,7 +650,7 @@ paths:
|
||||
"500":
|
||||
description: Error getting reservations
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/storage/request/{cid}":
|
||||
post:
|
||||
@ -659,7 +681,7 @@ paths:
|
||||
"404":
|
||||
description: Request ID not found
|
||||
"503":
|
||||
description: Purchasing is unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/storage/purchases":
|
||||
get:
|
||||
@ -676,7 +698,7 @@ paths:
|
||||
items:
|
||||
type: string
|
||||
"503":
|
||||
description: Purchasing is unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/storage/purchases/{id}":
|
||||
get:
|
||||
@ -702,7 +724,7 @@ paths:
|
||||
"404":
|
||||
description: Purchase not found
|
||||
"503":
|
||||
description: Purchasing is unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/node/spr":
|
||||
get:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -32,6 +32,7 @@ asyncchecksuite "Block Advertising and Discovery":
|
||||
peerStore: PeerCtxStore
|
||||
blockDiscovery: MockDiscovery
|
||||
discovery: DiscoveryEngine
|
||||
advertiser: Advertiser
|
||||
wallet: WalletRef
|
||||
network: BlockExcNetwork
|
||||
localStore: CacheStore
|
||||
@ -68,11 +69,17 @@ asyncchecksuite "Block Advertising and Discovery":
|
||||
pendingBlocks,
|
||||
minPeersPerBlock = 1)
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
engine = BlockExcEngine.new(
|
||||
localStore,
|
||||
wallet,
|
||||
network,
|
||||
discovery,
|
||||
advertiser,
|
||||
peerStore,
|
||||
pendingBlocks)
|
||||
|
||||
@ -200,11 +207,17 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
|
||||
pendingBlocks,
|
||||
minPeersPerBlock = 1)
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
engine = BlockExcEngine.new(
|
||||
localStore,
|
||||
wallet,
|
||||
network,
|
||||
discovery,
|
||||
advertiser,
|
||||
peerStore,
|
||||
pendingBlocks)
|
||||
networkStore = NetworkStore.new(engine, localStore)
|
||||
|
||||
@ -74,30 +74,6 @@ asyncchecksuite "Test Discovery Engine":
|
||||
await allFuturesThrowing(allFinished(wants)).wait(1.seconds)
|
||||
await discoveryEngine.stop()
|
||||
|
||||
test "Should Advertise Haves":
|
||||
var
|
||||
localStore = CacheStore.new(blocks.mapIt(it))
|
||||
discoveryEngine = DiscoveryEngine.new(
|
||||
localStore,
|
||||
peerStore,
|
||||
network,
|
||||
blockDiscovery,
|
||||
pendingBlocks,
|
||||
discoveryLoopSleep = 100.millis)
|
||||
haves = collect(initTable):
|
||||
for cid in @[manifestBlock.cid, manifest.treeCid]:
|
||||
{ cid: newFuture[void]() }
|
||||
|
||||
blockDiscovery.publishBlockProvideHandler =
|
||||
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
|
||||
if not haves[cid].finished:
|
||||
haves[cid].complete
|
||||
|
||||
await discoveryEngine.start()
|
||||
await allFuturesThrowing(
|
||||
allFinished(toSeq(haves.values))).wait(5.seconds)
|
||||
await discoveryEngine.stop()
|
||||
|
||||
test "Should queue discovery request":
|
||||
var
|
||||
localStore = CacheStore.new()
|
||||
@ -191,36 +167,3 @@ asyncchecksuite "Test Discovery Engine":
|
||||
|
||||
reqs.complete()
|
||||
await discoveryEngine.stop()
|
||||
|
||||
test "Should not request if there is already an inflight advertise request":
|
||||
var
|
||||
localStore = CacheStore.new()
|
||||
discoveryEngine = DiscoveryEngine.new(
|
||||
localStore,
|
||||
peerStore,
|
||||
network,
|
||||
blockDiscovery,
|
||||
pendingBlocks,
|
||||
discoveryLoopSleep = 100.millis,
|
||||
concurrentAdvReqs = 2)
|
||||
reqs = newFuture[void]()
|
||||
count = 0
|
||||
|
||||
blockDiscovery.publishBlockProvideHandler =
|
||||
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
|
||||
check cid == blocks[0].cid
|
||||
if count > 0:
|
||||
check false
|
||||
count.inc
|
||||
|
||||
await reqs # queue the request
|
||||
|
||||
await discoveryEngine.start()
|
||||
discoveryEngine.queueProvideBlocksReq(@[blocks[0].cid])
|
||||
await sleepAsync(200.millis)
|
||||
|
||||
discoveryEngine.queueProvideBlocksReq(@[blocks[0].cid])
|
||||
await sleepAsync(200.millis)
|
||||
|
||||
reqs.complete()
|
||||
await discoveryEngine.stop()
|
||||
|
||||
106
tests/codex/blockexchange/engine/testadvertiser.nim
Normal file
106
tests/codex/blockexchange/engine/testadvertiser.nim
Normal file
@ -0,0 +1,106 @@
|
||||
import std/sequtils
|
||||
import std/random
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/libp2p/routing_record
|
||||
import pkg/codexdht/discv5/protocol as discv5
|
||||
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/discovery
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/manifest
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
import ../../helpers/mockdiscovery
|
||||
import ../../examples
|
||||
|
||||
asyncchecksuite "Advertiser":
|
||||
var
|
||||
blockDiscovery: MockDiscovery
|
||||
localStore: BlockStore
|
||||
advertiser: Advertiser
|
||||
let
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = 123.NBytes,
|
||||
datasetSize = 234.NBytes)
|
||||
manifestBlk = Block.new(data = manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
|
||||
setup:
|
||||
blockDiscovery = MockDiscovery.new()
|
||||
localStore = CacheStore.new()
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
await advertiser.start()
|
||||
|
||||
teardown:
|
||||
await advertiser.stop()
|
||||
|
||||
test "blockStored should queue manifest Cid for advertising":
|
||||
(await localStore.putBlock(manifestBlk)).tryGet()
|
||||
|
||||
check:
|
||||
manifestBlk.cid in advertiser.advertiseQueue
|
||||
|
||||
test "blockStored should queue tree Cid for advertising":
|
||||
(await localStore.putBlock(manifestBlk)).tryGet()
|
||||
|
||||
check:
|
||||
manifest.treeCid in advertiser.advertiseQueue
|
||||
|
||||
test "blockStored should not queue non-manifest non-tree CIDs for discovery":
|
||||
let blk = bt.Block.example
|
||||
|
||||
(await localStore.putBlock(blk)).tryGet()
|
||||
|
||||
check:
|
||||
blk.cid notin advertiser.advertiseQueue
|
||||
|
||||
test "Should not queue if there is already an inflight advertise request":
|
||||
var
|
||||
reqs = newFuture[void]()
|
||||
manifestCount = 0
|
||||
treeCount = 0
|
||||
|
||||
blockDiscovery.publishBlockProvideHandler =
|
||||
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
|
||||
if cid == manifestBlk.cid:
|
||||
inc manifestCount
|
||||
if cid == manifest.treeCid:
|
||||
inc treeCount
|
||||
|
||||
await reqs # queue the request
|
||||
|
||||
(await localStore.putBlock(manifestBlk)).tryGet()
|
||||
(await localStore.putBlock(manifestBlk)).tryGet()
|
||||
|
||||
reqs.complete()
|
||||
check eventually manifestCount == 1
|
||||
check eventually treeCount == 1
|
||||
|
||||
test "Should advertise existing manifests and their trees":
|
||||
let
|
||||
newStore = CacheStore.new([manifestBlk])
|
||||
|
||||
await advertiser.stop()
|
||||
advertiser = Advertiser.new(
|
||||
newStore,
|
||||
blockDiscovery
|
||||
)
|
||||
await advertiser.start()
|
||||
|
||||
check eventually manifestBlk.cid in advertiser.advertiseQueue
|
||||
check eventually manifest.treeCid in advertiser.advertiseQueue
|
||||
|
||||
test "Stop should clear onBlockStored callback":
|
||||
await advertiser.stop()
|
||||
|
||||
check:
|
||||
localStore.onBlockStored.isNone()
|
||||
@ -78,11 +78,17 @@ asyncchecksuite "NetworkStore engine basic":
|
||||
blockDiscovery,
|
||||
pendingBlocks)
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
engine = BlockExcEngine.new(
|
||||
localStore,
|
||||
wallet,
|
||||
network,
|
||||
discovery,
|
||||
advertiser,
|
||||
peerStore,
|
||||
pendingBlocks)
|
||||
|
||||
@ -113,11 +119,17 @@ asyncchecksuite "NetworkStore engine basic":
|
||||
blockDiscovery,
|
||||
pendingBlocks)
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
engine = BlockExcEngine.new(
|
||||
localStore,
|
||||
wallet,
|
||||
network,
|
||||
discovery,
|
||||
advertiser,
|
||||
peerStore,
|
||||
pendingBlocks)
|
||||
|
||||
@ -139,6 +151,7 @@ asyncchecksuite "NetworkStore engine handlers":
|
||||
network: BlockExcNetwork
|
||||
engine: BlockExcEngine
|
||||
discovery: DiscoveryEngine
|
||||
advertiser: Advertiser
|
||||
peerCtx: BlockExcPeerCtx
|
||||
localStore: BlockStore
|
||||
blocks: seq[Block]
|
||||
@ -176,11 +189,17 @@ asyncchecksuite "NetworkStore engine handlers":
|
||||
blockDiscovery,
|
||||
pendingBlocks)
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
engine = BlockExcEngine.new(
|
||||
localStore,
|
||||
wallet,
|
||||
network,
|
||||
discovery,
|
||||
advertiser,
|
||||
peerStore,
|
||||
pendingBlocks)
|
||||
|
||||
@ -390,51 +409,6 @@ asyncchecksuite "NetworkStore engine handlers":
|
||||
discard await allFinished(pending)
|
||||
await allFuturesThrowing(cancellations.values().toSeq)
|
||||
|
||||
test "resolveBlocks should queue manifest CIDs for discovery":
|
||||
engine.network = BlockExcNetwork(
|
||||
request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc))
|
||||
|
||||
let
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = 123.NBytes,
|
||||
datasetSize = 234.NBytes
|
||||
)
|
||||
|
||||
let manifestBlk = Block.new(data = manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
let blks = @[manifestBlk]
|
||||
|
||||
await engine.resolveBlocks(blks)
|
||||
|
||||
check:
|
||||
manifestBlk.cid in engine.discovery.advertiseQueue
|
||||
|
||||
test "resolveBlocks should queue tree CIDs for discovery":
|
||||
engine.network = BlockExcNetwork(
|
||||
request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc))
|
||||
|
||||
let
|
||||
tCid = Cid.example
|
||||
delivery = BlockDelivery(blk: Block.example, address: BlockAddress(leaf: true, treeCid: tCid))
|
||||
|
||||
await engine.resolveBlocks(@[delivery])
|
||||
|
||||
check:
|
||||
tCid in engine.discovery.advertiseQueue
|
||||
|
||||
test "resolveBlocks should not queue non-manifest non-tree CIDs for discovery":
|
||||
engine.network = BlockExcNetwork(
|
||||
request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc))
|
||||
|
||||
let
|
||||
blkCid = Cid.example
|
||||
delivery = BlockDelivery(blk: Block.example, address: BlockAddress(leaf: false, cid: blkCid))
|
||||
|
||||
await engine.resolveBlocks(@[delivery])
|
||||
|
||||
check:
|
||||
blkCid notin engine.discovery.advertiseQueue
|
||||
|
||||
asyncchecksuite "Task Handler":
|
||||
var
|
||||
rng: Rng
|
||||
@ -448,6 +422,7 @@ asyncchecksuite "Task Handler":
|
||||
network: BlockExcNetwork
|
||||
engine: BlockExcEngine
|
||||
discovery: DiscoveryEngine
|
||||
advertiser: Advertiser
|
||||
localStore: BlockStore
|
||||
|
||||
peersCtx: seq[BlockExcPeerCtx]
|
||||
@ -481,11 +456,17 @@ asyncchecksuite "Task Handler":
|
||||
blockDiscovery,
|
||||
pendingBlocks)
|
||||
|
||||
advertiser = Advertiser.new(
|
||||
localStore,
|
||||
blockDiscovery
|
||||
)
|
||||
|
||||
engine = BlockExcEngine.new(
|
||||
localStore,
|
||||
wallet,
|
||||
network,
|
||||
discovery,
|
||||
advertiser,
|
||||
peerStore,
|
||||
pendingBlocks)
|
||||
peersCtx = @[]
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import ./engine/testengine
|
||||
import ./engine/testblockexc
|
||||
import ./engine/testpayments
|
||||
import ./engine/testadvertiser
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
||||
@ -38,6 +38,8 @@ type
|
||||
signer: Address
|
||||
subscriptions: Subscriptions
|
||||
config*: MarketplaceConfig
|
||||
canReserveSlot*: bool
|
||||
reserveSlotThrowError*: ?(ref MarketError)
|
||||
Fulfillment* = object
|
||||
requestId*: RequestId
|
||||
proof*: Groth16Proof
|
||||
@ -52,6 +54,7 @@ type
|
||||
onFulfillment: seq[FulfillmentSubscription]
|
||||
onSlotFilled: seq[SlotFilledSubscription]
|
||||
onSlotFreed: seq[SlotFreedSubscription]
|
||||
onSlotReservationsFull: seq[SlotReservationsFullSubscription]
|
||||
onRequestCancelled: seq[RequestCancelledSubscription]
|
||||
onRequestFailed: seq[RequestFailedSubscription]
|
||||
onProofSubmitted: seq[ProofSubmittedSubscription]
|
||||
@ -70,6 +73,9 @@ type
|
||||
SlotFreedSubscription* = ref object of Subscription
|
||||
market: MockMarket
|
||||
callback: OnSlotFreed
|
||||
SlotReservationsFullSubscription* = ref object of Subscription
|
||||
market: MockMarket
|
||||
callback: OnSlotReservationsFull
|
||||
RequestCancelledSubscription* = ref object of Subscription
|
||||
market: MockMarket
|
||||
requestId: ?RequestId
|
||||
@ -101,10 +107,11 @@ proc new*(_: type MockMarket): MockMarket =
|
||||
proofs: ProofConfig(
|
||||
period: 10.u256,
|
||||
timeout: 5.u256,
|
||||
downtime: 64.uint8
|
||||
downtime: 64.uint8,
|
||||
downtimeProduct: 67.uint8
|
||||
)
|
||||
)
|
||||
MockMarket(signer: Address.example, config: config)
|
||||
MockMarket(signer: Address.example, config: config, canReserveSlot: true)
|
||||
|
||||
method getSigner*(market: MockMarket): Future[Address] {.async.} =
|
||||
return market.signer
|
||||
@ -199,6 +206,15 @@ proc emitSlotFreed*(market: MockMarket,
|
||||
for subscription in subscriptions:
|
||||
subscription.callback(requestId, slotIndex)
|
||||
|
||||
proc emitSlotReservationsFull*(
|
||||
market: MockMarket,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256) =
|
||||
|
||||
var subscriptions = market.subscriptions.onSlotReservationsFull
|
||||
for subscription in subscriptions:
|
||||
subscription.callback(requestId, slotIndex)
|
||||
|
||||
proc emitRequestCancelled*(market: MockMarket, requestId: RequestId) =
|
||||
var subscriptions = market.subscriptions.onRequestCancelled
|
||||
for subscription in subscriptions:
|
||||
@ -302,6 +318,29 @@ method canProofBeMarkedAsMissing*(market: MockMarket,
|
||||
period: Period): Future[bool] {.async.} =
|
||||
return market.canBeMarkedAsMissing.contains(id)
|
||||
|
||||
method reserveSlot*(
|
||||
market: MockMarket,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256) {.async.} =
|
||||
|
||||
if error =? market.reserveSlotThrowError:
|
||||
raise error
|
||||
|
||||
method canReserveSlot*(
|
||||
market: MockMarket,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256): Future[bool] {.async.} =
|
||||
|
||||
return market.canReserveSlot
|
||||
|
||||
func setCanReserveSlot*(market: MockMarket, canReserveSlot: bool) =
|
||||
market.canReserveSlot = canReserveSlot
|
||||
|
||||
func setReserveSlotThrowError*(
|
||||
market: MockMarket, error: ?(ref MarketError)) =
|
||||
|
||||
market.reserveSlotThrowError = error
|
||||
|
||||
method subscribeRequests*(market: MockMarket,
|
||||
callback: OnRequest):
|
||||
Future[Subscription] {.async.} =
|
||||
@ -363,6 +402,15 @@ method subscribeSlotFreed*(market: MockMarket,
|
||||
market.subscriptions.onSlotFreed.add(subscription)
|
||||
return subscription
|
||||
|
||||
method subscribeSlotReservationsFull*(
|
||||
market: MockMarket,
|
||||
callback: OnSlotReservationsFull): Future[Subscription] {.async.} =
|
||||
|
||||
let subscription =
|
||||
SlotReservationsFullSubscription(market: market, callback: callback)
|
||||
market.subscriptions.onSlotReservationsFull.add(subscription)
|
||||
return subscription
|
||||
|
||||
method subscribeRequestCancelled*(market: MockMarket,
|
||||
callback: OnRequestCancelled):
|
||||
Future[Subscription] {.async.} =
|
||||
@ -419,16 +467,21 @@ method subscribeProofSubmission*(mock: MockMarket,
|
||||
mock.subscriptions.onProofSubmitted.add(subscription)
|
||||
return subscription
|
||||
|
||||
method queryPastStorageRequests*(market: MockMarket,
|
||||
blocksAgo: int):
|
||||
Future[seq[PastStorageRequest]] {.async.} =
|
||||
# MockMarket does not have the concept of blocks, so simply return all
|
||||
# previous events
|
||||
return market.requested.map(request =>
|
||||
PastStorageRequest(requestId: request.id,
|
||||
method queryPastEvents*[T: MarketplaceEvent](
|
||||
market: MockMarket,
|
||||
_: type T,
|
||||
blocksAgo: int): Future[seq[T]] {.async.} =
|
||||
|
||||
if T of StorageRequested:
|
||||
return market.requested.map(request =>
|
||||
StorageRequested(requestId: request.id,
|
||||
ask: request.ask,
|
||||
expiry: request.expiry)
|
||||
)
|
||||
)
|
||||
elif T of SlotFilled:
|
||||
return market.filled.map(slot =>
|
||||
SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex)
|
||||
)
|
||||
|
||||
method unsubscribe*(subscription: RequestSubscription) {.async.} =
|
||||
subscription.market.subscriptions.onRequest.keepItIf(it != subscription)
|
||||
@ -450,3 +503,6 @@ method unsubscribe*(subscription: RequestFailedSubscription) {.async.} =
|
||||
|
||||
method unsubscribe*(subscription: ProofSubmittedSubscription) {.async.} =
|
||||
subscription.market.subscriptions.onProofSubmitted.keepItIf(it != subscription)
|
||||
|
||||
method unsubscribe*(subscription: SlotReservationsFullSubscription) {.async.} =
|
||||
subscription.market.subscriptions.onSlotReservationsFull.keepItIf(it != subscription)
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
## those terms.
|
||||
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
import pkg/chronos
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
@ -24,33 +25,28 @@ type
|
||||
|
||||
testBlockExpirations*: seq[BlockExpiration]
|
||||
getBlockExpirationsThrows*: bool
|
||||
iteratorIndex: int
|
||||
|
||||
method delBlock*(self: MockRepoStore, cid: Cid): Future[?!void] {.async.} =
|
||||
self.delBlockCids.add(cid)
|
||||
self.testBlockExpirations = self.testBlockExpirations.filterIt(it.cid != cid)
|
||||
dec self.iteratorIndex
|
||||
return success()
|
||||
|
||||
method getBlockExpirations*(self: MockRepoStore, maxNumber: int, offset: int): Future[?!AsyncIter[?BlockExpiration]] {.async.} =
|
||||
method getBlockExpirations*(self: MockRepoStore, maxNumber: int, offset: int): Future[?!AsyncIter[BlockExpiration]] {.async.} =
|
||||
if self.getBlockExpirationsThrows:
|
||||
raise new CatchableError
|
||||
|
||||
self.getBeMaxNumber = maxNumber
|
||||
self.getBeOffset = offset
|
||||
|
||||
var iter = AsyncIter[?BlockExpiration]()
|
||||
let
|
||||
testBlockExpirationsCpy = @(self.testBlockExpirations)
|
||||
limit = min(offset + maxNumber, len(testBlockExpirationsCpy))
|
||||
|
||||
self.iteratorIndex = offset
|
||||
var numberLeft = maxNumber
|
||||
proc next(): Future[?BlockExpiration] {.async.} =
|
||||
if numberLeft > 0 and self.iteratorIndex >= 0 and self.iteratorIndex < len(self.testBlockExpirations):
|
||||
dec numberLeft
|
||||
let selectedBlock = self.testBlockExpirations[self.iteratorIndex]
|
||||
inc self.iteratorIndex
|
||||
return selectedBlock.some
|
||||
iter.finish
|
||||
return BlockExpiration.none
|
||||
let
|
||||
iter1 = AsyncIter[int].new(offset..<limit)
|
||||
iter2 = map[int, BlockExpiration](iter1,
|
||||
proc (i: int): Future[BlockExpiration] {.async.} =
|
||||
testBlockExpirationsCpy[i]
|
||||
)
|
||||
|
||||
iter.next = next
|
||||
return success iter
|
||||
success(iter2)
|
||||
|
||||
44
tests/codex/helpers/mockreservations.nim
Normal file
44
tests/codex/helpers/mockreservations.nim
Normal file
@ -0,0 +1,44 @@
|
||||
import pkg/chronos
|
||||
import pkg/codex/sales
|
||||
import pkg/codex/stores
|
||||
import pkg/questionable/results
|
||||
|
||||
type
|
||||
MockReservations* = ref object of Reservations
|
||||
createReservationThrowBytesOutOfBoundsError: bool
|
||||
createReservationThrowError: ?(ref CatchableError)
|
||||
|
||||
proc new*(
|
||||
T: type MockReservations,
|
||||
repo: RepoStore
|
||||
): MockReservations =
|
||||
## Create a mock clock instance
|
||||
MockReservations(availabilityLock: newAsyncLock(), repo: repo)
|
||||
|
||||
proc setCreateReservationThrowBytesOutOfBoundsError*(
|
||||
self: MockReservations, flag: bool) =
|
||||
|
||||
self.createReservationThrowBytesOutOfBoundsError = flag
|
||||
|
||||
proc setCreateReservationThrowError*(
|
||||
self: MockReservations, error: ?(ref CatchableError)) =
|
||||
|
||||
self.createReservationThrowError = error
|
||||
|
||||
method createReservation*(
|
||||
self: MockReservations,
|
||||
availabilityId: AvailabilityId,
|
||||
slotSize: UInt256,
|
||||
requestId: RequestId,
|
||||
slotIndex: UInt256): Future[?!Reservation] {.async.} =
|
||||
if self.createReservationThrowBytesOutOfBoundsError:
|
||||
let error = newException(
|
||||
BytesOutOfBoundsError,
|
||||
"trying to reserve an amount of bytes that is greater than the total size of the Availability")
|
||||
return failure(error)
|
||||
|
||||
elif error =? self.createReservationThrowError:
|
||||
return failure(error)
|
||||
|
||||
return await procCall createReservation(Reservations(self), availabilityId, slotSize, requestId, slotIndex)
|
||||
|
||||
@ -40,8 +40,9 @@ proc generateNodes*(
|
||||
localStore = CacheStore.new(blocks.mapIt( it ))
|
||||
peerStore = PeerCtxStore.new()
|
||||
pendingBlocks = PendingBlocksManager.new()
|
||||
advertiser = Advertiser.new(localStore, discovery)
|
||||
blockDiscovery = DiscoveryEngine.new(localStore, peerStore, network, discovery, pendingBlocks)
|
||||
engine = BlockExcEngine.new(localStore, wallet, network, blockDiscovery, peerStore, pendingBlocks)
|
||||
engine = BlockExcEngine.new(localStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks)
|
||||
networkStore = NetworkStore.new(engine, localStore)
|
||||
|
||||
switch.mount(network)
|
||||
|
||||
@ -5,7 +5,6 @@ import std/cpuinfo
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/taskpools
|
||||
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
@ -83,6 +82,7 @@ template setupAndTearDown*() {.dirty.} =
|
||||
peerStore: PeerCtxStore
|
||||
pendingBlocks: PendingBlocksManager
|
||||
discovery: DiscoveryEngine
|
||||
advertiser: Advertiser
|
||||
taskpool: Taskpool
|
||||
|
||||
let
|
||||
@ -110,7 +110,8 @@ template setupAndTearDown*() {.dirty.} =
|
||||
peerStore = PeerCtxStore.new()
|
||||
pendingBlocks = PendingBlocksManager.new()
|
||||
discovery = DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks)
|
||||
engine = BlockExcEngine.new(localStore, wallet, network, discovery, peerStore, pendingBlocks)
|
||||
advertiser = Advertiser.new(localStore, blockDiscovery)
|
||||
engine = BlockExcEngine.new(localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks)
|
||||
store = NetworkStore.new(engine, localStore)
|
||||
taskpool = Taskpool.new(num_threads = countProcessors())
|
||||
node = CodexNodeRef.new(
|
||||
@ -121,8 +122,6 @@ template setupAndTearDown*() {.dirty.} =
|
||||
discovery = blockDiscovery,
|
||||
taskpool = taskpool)
|
||||
|
||||
await node.start()
|
||||
|
||||
teardown:
|
||||
close(file)
|
||||
await node.stop()
|
||||
|
||||
@ -9,6 +9,7 @@ import std/cpuinfo
|
||||
import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
import pkg/datastore
|
||||
import pkg/datastore/typedds
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/stint
|
||||
@ -32,6 +33,7 @@ import pkg/codex/discovery
|
||||
import pkg/codex/erasure
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/stores/repostore/coders
|
||||
import pkg/codex/utils/asynciter
|
||||
import pkg/codex/indexingstrategy
|
||||
|
||||
@ -110,10 +112,11 @@ asyncchecksuite "Test Node - Host contracts":
|
||||
for index in 0..<manifest.blocksCount:
|
||||
let
|
||||
blk = (await localStore.getBlock(manifest.treeCid, index)).tryGet
|
||||
expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
||||
expiry = await localStoreMetaDs.get(expiryKey)
|
||||
key = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
||||
bytes = (await localStoreMetaDs.get(key)).tryGet
|
||||
blkMd = BlockMetadata.decode(bytes).tryGet
|
||||
|
||||
check (expiry.tryGet).toSecondsSince1970 == expectedExpiry
|
||||
check blkMd.expiry == expectedExpiry
|
||||
|
||||
test "onStore callback is set":
|
||||
check sales.onStore.isSome
|
||||
@ -131,7 +134,7 @@ asyncchecksuite "Test Node - Host contracts":
|
||||
return success()
|
||||
|
||||
(await onStore(request, 1.u256, onBlocks)).tryGet()
|
||||
check fetchedBytes == 262144
|
||||
check fetchedBytes == 12 * DefaultBlockSize.uint
|
||||
|
||||
let indexer = verifiable.protectedStrategy.init(
|
||||
0, verifiable.numSlotBlocks() - 1, verifiable.numSlots)
|
||||
@ -139,7 +142,8 @@ asyncchecksuite "Test Node - Host contracts":
|
||||
for index in indexer.getIndicies(1):
|
||||
let
|
||||
blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet
|
||||
expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
||||
expiry = await localStoreMetaDs.get(expiryKey)
|
||||
key = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
||||
bytes = (await localStoreMetaDs.get(key)).tryGet
|
||||
blkMd = BlockMetadata.decode(bytes).tryGet
|
||||
|
||||
check (expiry.tryGet).toSecondsSince1970 == request.expiry.toSecondsSince1970
|
||||
check blkMd.expiry == request.expiry.toSecondsSince1970
|
||||
|
||||
@ -9,6 +9,7 @@ import std/cpuinfo
|
||||
import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
import pkg/datastore
|
||||
import pkg/datastore/typedds
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/stint
|
||||
@ -48,6 +49,9 @@ privateAccess(CodexNodeRef) # enable access to private fields
|
||||
asyncchecksuite "Test Node - Basic":
|
||||
setupAndTearDown()
|
||||
|
||||
setup:
|
||||
await node.start()
|
||||
|
||||
test "Fetch Manifest":
|
||||
let
|
||||
manifest = await storeDataGetManifest(localStore, chunker)
|
||||
|
||||
@ -39,7 +39,8 @@ asyncchecksuite "sales state 'ignored'":
|
||||
agent.onCleanUp = onCleanUp
|
||||
state = SaleIgnored.new()
|
||||
|
||||
test "calls onCleanUp with returnBytes = false and reprocessSlot = true":
|
||||
test "calls onCleanUp with values assigned to SaleIgnored":
|
||||
state = SaleIgnored(reprocessSlot: true, returnBytes: true)
|
||||
discard await state.run(agent)
|
||||
check eventually returnBytesWas == false
|
||||
check eventually returnBytesWas == true
|
||||
check eventually reprocessSlotWas == true
|
||||
|
||||
@ -1,20 +1,66 @@
|
||||
import std/unittest
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
import pkg/datastore
|
||||
import pkg/stew/byteutils
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/sales/states/downloading
|
||||
import pkg/codex/sales/states/preparing
|
||||
import pkg/codex/sales/states/slotreserving
|
||||
import pkg/codex/sales/states/cancelled
|
||||
import pkg/codex/sales/states/failed
|
||||
import pkg/codex/sales/states/filled
|
||||
import pkg/codex/sales/states/ignored
|
||||
import pkg/codex/sales/states/errored
|
||||
import pkg/codex/sales/salesagent
|
||||
import pkg/codex/sales/salescontext
|
||||
import pkg/codex/sales/reservations
|
||||
import pkg/codex/stores/repostore
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
import ../../examples
|
||||
import ../../helpers/mockmarket
|
||||
import ../../helpers/mockreservations
|
||||
import ../../helpers/mockclock
|
||||
|
||||
suite "sales state 'preparing'":
|
||||
|
||||
asyncchecksuite "sales state 'preparing'":
|
||||
let request = StorageRequest.example
|
||||
let slotIndex = (request.ask.slots div 2).u256
|
||||
let market = MockMarket.new()
|
||||
let clock = MockClock.new()
|
||||
var agent: SalesAgent
|
||||
var state: SalePreparing
|
||||
var repo: RepoStore
|
||||
var availability: Availability
|
||||
var context: SalesContext
|
||||
var reservations: MockReservations
|
||||
|
||||
setup:
|
||||
availability = Availability(
|
||||
totalSize: request.ask.slotSize + 100.u256,
|
||||
freeSize: request.ask.slotSize + 100.u256,
|
||||
duration: request.ask.duration + 60.u256,
|
||||
minPrice: request.ask.pricePerSlot - 10.u256,
|
||||
maxCollateral: request.ask.collateral + 400.u256
|
||||
)
|
||||
let repoDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
let metaDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
repo = RepoStore.new(repoDs, metaDs)
|
||||
await repo.start()
|
||||
|
||||
state = SalePreparing.new()
|
||||
context = SalesContext(
|
||||
market: market,
|
||||
clock: clock
|
||||
)
|
||||
|
||||
reservations = MockReservations.new(repo)
|
||||
context.reservations = reservations
|
||||
agent = newSalesAgent(context,
|
||||
request.id,
|
||||
slotIndex,
|
||||
request.some)
|
||||
|
||||
teardown:
|
||||
await repo.stop()
|
||||
|
||||
test "switches to cancelled state when request expires":
|
||||
let next = state.onCancelled(request)
|
||||
@ -27,3 +73,44 @@ suite "sales state 'preparing'":
|
||||
test "switches to filled state when slot is filled":
|
||||
let next = state.onSlotFilled(request.id, slotIndex)
|
||||
check !next of SaleFilled
|
||||
|
||||
proc createAvailability() {.async.} =
|
||||
let a = await reservations.createAvailability(
|
||||
availability.totalSize,
|
||||
availability.duration,
|
||||
availability.minPrice,
|
||||
availability.maxCollateral
|
||||
)
|
||||
availability = a.get
|
||||
|
||||
test "run switches to ignored when no availability":
|
||||
let next = !(await state.run(agent))
|
||||
check next of SaleIgnored
|
||||
let ignored = SaleIgnored(next)
|
||||
check ignored.reprocessSlot
|
||||
check ignored.returnBytes == false
|
||||
|
||||
test "run switches to slot reserving state after reservation created":
|
||||
await createAvailability()
|
||||
let next = await state.run(agent)
|
||||
check !next of SaleSlotReserving
|
||||
|
||||
test "run switches to ignored when reserve fails with BytesOutOfBounds":
|
||||
await createAvailability()
|
||||
reservations.setCreateReservationThrowBytesOutOfBoundsError(true)
|
||||
|
||||
let next = !(await state.run(agent))
|
||||
check next of SaleIgnored
|
||||
let ignored = SaleIgnored(next)
|
||||
check ignored.reprocessSlot
|
||||
check ignored.returnBytes == false
|
||||
|
||||
test "run switches to errored when reserve fails with other error":
|
||||
await createAvailability()
|
||||
let error = newException(CatchableError, "some error")
|
||||
reservations.setCreateReservationThrowError(some error)
|
||||
|
||||
let next = !(await state.run(agent))
|
||||
check next of SaleErrored
|
||||
let errored = SaleErrored(next)
|
||||
check errored.error == error
|
||||
|
||||
73
tests/codex/sales/states/testslotreserving.nim
Normal file
73
tests/codex/sales/states/testslotreserving.nim
Normal file
@ -0,0 +1,73 @@
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/sales/states/slotreserving
|
||||
import pkg/codex/sales/states/downloading
|
||||
import pkg/codex/sales/states/cancelled
|
||||
import pkg/codex/sales/states/failed
|
||||
import pkg/codex/sales/states/filled
|
||||
import pkg/codex/sales/states/ignored
|
||||
import pkg/codex/sales/states/errored
|
||||
import pkg/codex/sales/salesagent
|
||||
import pkg/codex/sales/salescontext
|
||||
import pkg/codex/sales/reservations
|
||||
import pkg/codex/stores/repostore
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
import ../../examples
|
||||
import ../../helpers/mockmarket
|
||||
import ../../helpers/mockreservations
|
||||
import ../../helpers/mockclock
|
||||
|
||||
asyncchecksuite "sales state 'SlotReserving'":
|
||||
let request = StorageRequest.example
|
||||
let slotIndex = (request.ask.slots div 2).u256
|
||||
var market: MockMarket
|
||||
var clock: MockClock
|
||||
var agent: SalesAgent
|
||||
var state: SaleSlotReserving
|
||||
var context: SalesContext
|
||||
|
||||
setup:
|
||||
market = MockMarket.new()
|
||||
clock = MockClock.new()
|
||||
|
||||
state = SaleSlotReserving.new()
|
||||
context = SalesContext(
|
||||
market: market,
|
||||
clock: clock
|
||||
)
|
||||
|
||||
agent = newSalesAgent(context,
|
||||
request.id,
|
||||
slotIndex,
|
||||
request.some)
|
||||
|
||||
test "switches to cancelled state when request expires":
|
||||
let next = state.onCancelled(request)
|
||||
check !next of SaleCancelled
|
||||
|
||||
test "switches to failed state when request fails":
|
||||
let next = state.onFailed(request)
|
||||
check !next of SaleFailed
|
||||
|
||||
test "switches to filled state when slot is filled":
|
||||
let next = state.onSlotFilled(request.id, slotIndex)
|
||||
check !next of SaleFilled
|
||||
|
||||
test "run switches to downloading when slot successfully reserved":
|
||||
let next = await state.run(agent)
|
||||
check !next of SaleDownloading
|
||||
|
||||
test "run switches to ignored when slot reservation not allowed":
|
||||
market.setCanReserveSlot(false)
|
||||
let next = await state.run(agent)
|
||||
check !next of SaleIgnored
|
||||
|
||||
test "run switches to errored when slot reservation errors":
|
||||
let error = newException(MarketError, "some error")
|
||||
market.setReserveSlotThrowError(some error)
|
||||
let next = !(await state.run(agent))
|
||||
check next of SaleErrored
|
||||
let errored = SaleErrored(next)
|
||||
check errored.error == error
|
||||
@ -1,4 +1,5 @@
|
||||
import std/random
|
||||
import std/sequtils
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
@ -6,6 +7,7 @@ import pkg/chronos
|
||||
import pkg/datastore
|
||||
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/errors
|
||||
import pkg/codex/sales
|
||||
import pkg/codex/utils/json
|
||||
|
||||
@ -13,6 +15,8 @@ import ../../asynctest
|
||||
import ../examples
|
||||
import ../helpers
|
||||
|
||||
const CONCURRENCY_TESTS_COUNT = 1000
|
||||
|
||||
asyncchecksuite "Reservations module":
|
||||
var
|
||||
repo: RepoStore
|
||||
@ -73,9 +77,9 @@ asyncchecksuite "Reservations module":
|
||||
check availability.id != AvailabilityId.default
|
||||
|
||||
test "creating availability reserves bytes in repo":
|
||||
let orig = repo.available
|
||||
let orig = repo.available.uint
|
||||
let availability = createAvailability()
|
||||
check repo.available == (orig.u256 - availability.freeSize).truncate(uint)
|
||||
check repo.available.uint == (orig.u256 - availability.freeSize).truncate(uint)
|
||||
|
||||
test "can get all availabilities":
|
||||
let availability1 = createAvailability()
|
||||
@ -148,6 +152,39 @@ asyncchecksuite "Reservations module":
|
||||
check created.isErr
|
||||
check created.error of BytesOutOfBoundsError
|
||||
|
||||
test "cannot create reservation larger than availability size - concurrency test":
|
||||
proc concurrencyTest(): Future[void] {.async.} =
|
||||
let availability = createAvailability()
|
||||
let one = reservations.createReservation(
|
||||
availability.id,
|
||||
availability.totalSize - 1,
|
||||
RequestId.example,
|
||||
UInt256.example
|
||||
)
|
||||
|
||||
let two = reservations.createReservation(
|
||||
availability.id,
|
||||
availability.totalSize,
|
||||
RequestId.example,
|
||||
UInt256.example
|
||||
)
|
||||
|
||||
let oneResult = await one
|
||||
let twoResult = await two
|
||||
|
||||
check oneResult.isErr or twoResult.isErr
|
||||
if oneResult.isErr:
|
||||
check oneResult.error of BytesOutOfBoundsError
|
||||
if twoResult.isErr:
|
||||
check twoResult.error of BytesOutOfBoundsError
|
||||
|
||||
var futures: seq[Future[void]]
|
||||
for _ in 1..CONCURRENCY_TESTS_COUNT:
|
||||
futures.add(concurrencyTest())
|
||||
|
||||
await allFuturesThrowing(futures)
|
||||
|
||||
|
||||
test "creating reservation reduces availability size":
|
||||
let availability = createAvailability()
|
||||
let orig = availability.freeSize
|
||||
@ -211,7 +248,7 @@ asyncchecksuite "Reservations module":
|
||||
|
||||
check updated.freeSize > orig
|
||||
check (updated.freeSize - orig) == 200.u256
|
||||
check (repo.quotaReservedBytes - origQuota) == 200
|
||||
check (repo.quotaReservedBytes - origQuota) == 200.NBytes
|
||||
|
||||
test "update releases quota when lowering size":
|
||||
let
|
||||
@ -220,7 +257,7 @@ asyncchecksuite "Reservations module":
|
||||
availability.totalSize = availability.totalSize - 100
|
||||
|
||||
check isOk await reservations.update(availability)
|
||||
check (origQuota - repo.quotaReservedBytes) == 100
|
||||
check (origQuota - repo.quotaReservedBytes) == 100.NBytes
|
||||
|
||||
test "update reserves quota when growing size":
|
||||
let
|
||||
@ -229,7 +266,7 @@ asyncchecksuite "Reservations module":
|
||||
availability.totalSize = availability.totalSize + 100
|
||||
|
||||
check isOk await reservations.update(availability)
|
||||
check (repo.quotaReservedBytes - origQuota) == 100
|
||||
check (repo.quotaReservedBytes - origQuota) == 100.NBytes
|
||||
|
||||
test "reservation can be partially released":
|
||||
let availability = createAvailability()
|
||||
@ -333,17 +370,17 @@ asyncchecksuite "Reservations module":
|
||||
check got.error of NotExistsError
|
||||
|
||||
test "can get available bytes in repo":
|
||||
check reservations.available == DefaultQuotaBytes
|
||||
check reservations.available == DefaultQuotaBytes.uint
|
||||
|
||||
test "reports quota available to be reserved":
|
||||
check reservations.hasAvailable(DefaultQuotaBytes - 1)
|
||||
check reservations.hasAvailable(DefaultQuotaBytes.uint - 1)
|
||||
|
||||
test "reports quota not available to be reserved":
|
||||
check not reservations.hasAvailable(DefaultQuotaBytes + 1)
|
||||
check not reservations.hasAvailable(DefaultQuotaBytes.uint + 1)
|
||||
|
||||
test "fails to create availability with size that is larger than available quota":
|
||||
let created = await reservations.createAvailability(
|
||||
(DefaultQuotaBytes + 1).u256,
|
||||
(DefaultQuotaBytes.uint + 1).u256,
|
||||
UInt256.example,
|
||||
UInt256.example,
|
||||
UInt256.example
|
||||
|
||||
@ -2,7 +2,7 @@ import std/sequtils
|
||||
import std/sugar
|
||||
import std/times
|
||||
import pkg/chronos
|
||||
import pkg/datastore
|
||||
import pkg/datastore/typedds
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/sales
|
||||
@ -270,6 +270,12 @@ asyncchecksuite "Sales":
|
||||
let expected = SlotQueueItem.init(request1, 1'u16)
|
||||
check always (not itemsProcessed.contains(expected))
|
||||
|
||||
test "removes slot index from slot queue once SlotReservationsFull emitted":
|
||||
let request1 = await addRequestToSaturatedQueue()
|
||||
market.emitSlotReservationsFull(request1.id, 1.u256)
|
||||
let expected = SlotQueueItem.init(request1, 1'u16)
|
||||
check always (not itemsProcessed.contains(expected))
|
||||
|
||||
test "adds slot index to slot queue once SlotFreed emitted":
|
||||
queue.onProcessSlot = proc(item: SlotQueueItem, done: Future[void]) {.async.} =
|
||||
itemsProcessed.add item
|
||||
@ -323,8 +329,7 @@ asyncchecksuite "Sales":
|
||||
slot: UInt256,
|
||||
onBatch: BatchProc): Future[?!void] {.async.} =
|
||||
let blk = bt.Block.new( @[1.byte] ).get
|
||||
onBatch( blk.repeat(request.ask.slotSize.truncate(int)) )
|
||||
return success()
|
||||
await onBatch( blk.repeat(request.ask.slotSize.truncate(int)) )
|
||||
|
||||
createAvailability()
|
||||
await market.requestStorage(request)
|
||||
@ -337,8 +342,8 @@ asyncchecksuite "Sales":
|
||||
onBatch: BatchProc): Future[?!void] {.async.} =
|
||||
slotIndex = slot
|
||||
let blk = bt.Block.new( @[1.byte] ).get
|
||||
onBatch(@[ blk ])
|
||||
return success()
|
||||
await onBatch(@[ blk ])
|
||||
|
||||
let sold = newFuture[void]()
|
||||
sales.onSale = proc(request: StorageRequest, slotIndex: UInt256) =
|
||||
sold.complete()
|
||||
|
||||
@ -524,7 +524,7 @@ suite "Slot queue":
|
||||
request.ask,
|
||||
request.expiry,
|
||||
seen = true)
|
||||
queue.push(item)
|
||||
check queue.push(item).isOk
|
||||
check eventually queue.paused
|
||||
check onProcessSlotCalledWith.len == 0
|
||||
|
||||
@ -534,7 +534,7 @@ suite "Slot queue":
|
||||
|
||||
let request = StorageRequest.example
|
||||
var items = SlotQueueItem.init(request)
|
||||
queue.push(items)
|
||||
check queue.push(items).isOk
|
||||
# check all items processed
|
||||
check eventually queue.len == 0
|
||||
|
||||
@ -546,7 +546,7 @@ suite "Slot queue":
|
||||
request.expiry,
|
||||
seen = true)
|
||||
check queue.paused
|
||||
queue.push(item0)
|
||||
check queue.push(item0).isOk
|
||||
check queue.paused
|
||||
|
||||
test "paused queue waits for unpause before continuing processing":
|
||||
@ -558,7 +558,7 @@ suite "Slot queue":
|
||||
seen = false)
|
||||
check queue.paused
|
||||
# push causes unpause
|
||||
queue.push(item)
|
||||
check queue.push(item).isOk
|
||||
# check all items processed
|
||||
check eventually onProcessSlotCalledWith == @[
|
||||
(item.requestId, item.slotIndex),
|
||||
@ -576,8 +576,8 @@ suite "Slot queue":
|
||||
request.ask,
|
||||
request.expiry,
|
||||
seen = true)
|
||||
queue.push(item0)
|
||||
queue.push(item1)
|
||||
check queue.push(item0).isOk
|
||||
check queue.push(item1).isOk
|
||||
check queue[0].seen
|
||||
check queue[1].seen
|
||||
|
||||
|
||||
@ -6,5 +6,10 @@ import ./states/testinitialproving
|
||||
import ./states/testfilled
|
||||
import ./states/testproving
|
||||
import ./states/testsimulatedproving
|
||||
import ./states/testcancelled
|
||||
import ./states/testerrored
|
||||
import ./states/testignored
|
||||
import ./states/testpreparing
|
||||
import ./states/testslotreserving
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
||||
@ -17,21 +17,6 @@ import pkg/codex/utils/json
|
||||
|
||||
export types
|
||||
|
||||
func fromCircomData*(_: type Poseidon2Hash, cellData: seq[byte]): seq[Poseidon2Hash] =
|
||||
var
|
||||
pos = 0
|
||||
cellElms: seq[Bn254Fr]
|
||||
while pos < cellData.len:
|
||||
var
|
||||
step = 32
|
||||
offset = min(pos + step, cellData.len)
|
||||
data = cellData[pos..<offset]
|
||||
let ff = Bn254Fr.fromBytes(data.toArray32).get
|
||||
cellElms.add(ff)
|
||||
pos += data.len
|
||||
|
||||
cellElms
|
||||
|
||||
func toJsonDecimal*(big: BigInt[254]): string =
|
||||
let s = big.toDecimal.strip( leading = true, trailing = false, chars = {'0'} )
|
||||
if s.len == 0: "0" else: s
|
||||
@ -78,13 +63,16 @@ func toJson*(input: ProofInputs[Poseidon2Hash]): JsonNode =
|
||||
"slotRoot": input.slotRoot.toDecimal,
|
||||
"slotProof": input.slotProof.mapIt( it.toBig.toJsonDecimal ),
|
||||
"cellData": input.samples.mapIt(
|
||||
toSeq( it.cellData.elements(Poseidon2Hash) ).mapIt( it.toBig.toJsonDecimal )
|
||||
it.cellData.mapIt( it.toBig.toJsonDecimal )
|
||||
),
|
||||
"merklePaths": input.samples.mapIt(
|
||||
it.merklePaths.mapIt( it.toBig.toJsonDecimal )
|
||||
)
|
||||
}
|
||||
|
||||
func toJson*(input: NormalizedProofInputs[Poseidon2Hash]): JsonNode =
|
||||
toJson(ProofInputs[Poseidon2Hash](input))
|
||||
|
||||
func jsonToProofInput*(_: type Poseidon2Hash, inputJson: JsonNode): ProofInputs[Poseidon2Hash] =
|
||||
let
|
||||
cellData =
|
||||
@ -93,10 +81,12 @@ func jsonToProofInput*(_: type Poseidon2Hash, inputJson: JsonNode): ProofInputs[
|
||||
block:
|
||||
var
|
||||
big: BigInt[256]
|
||||
data = newSeq[byte](big.bits div 8)
|
||||
hash: Poseidon2Hash
|
||||
data: array[32, byte]
|
||||
assert bool(big.fromDecimal( it.str ))
|
||||
data.marshal(big, littleEndian)
|
||||
data
|
||||
assert data.marshal(big, littleEndian)
|
||||
|
||||
Poseidon2Hash.fromBytes(data).get
|
||||
).concat # flatten out elements
|
||||
)
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@ suite "Test Sampler - control samples":
|
||||
proofInput.nCellsPerSlot,
|
||||
sample.merklePaths[5..<9]).tryGet
|
||||
|
||||
cellData = Poseidon2Hash.fromCircomData(sample.cellData)
|
||||
cellData = sample.cellData
|
||||
cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
|
||||
slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
|
||||
|
||||
@ -158,7 +158,7 @@ suite "Test Sampler":
|
||||
nSlotCells,
|
||||
sample.merklePaths[5..<sample.merklePaths.len]).tryGet
|
||||
|
||||
cellData = Poseidon2Hash.fromCircomData(sample.cellData)
|
||||
cellData = sample.cellData
|
||||
cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
|
||||
slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
|
||||
|
||||
|
||||
103
tests/codex/slots/testbackendfactory.nim
Normal file
103
tests/codex/slots/testbackendfactory.nim
Normal file
@ -0,0 +1,103 @@
|
||||
import os
|
||||
import ../../asynctest
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/confutils/defs
|
||||
import pkg/codex/conf
|
||||
import pkg/codex/slots/proofs/backends
|
||||
import pkg/codex/slots/proofs/backendfactory
|
||||
import pkg/codex/slots/proofs/backendutils
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
|
||||
type
|
||||
BackendUtilsMock = ref object of BackendUtils
|
||||
argR1csFile: string
|
||||
argWasmFile: string
|
||||
argZKeyFile: string
|
||||
|
||||
method initializeCircomBackend*(
|
||||
self: BackendUtilsMock,
|
||||
r1csFile: string,
|
||||
wasmFile: string,
|
||||
zKeyFile: string
|
||||
): AnyBackend =
|
||||
self.argR1csFile = r1csFile
|
||||
self.argWasmFile = wasmFile
|
||||
self.argZKeyFile = zKeyFile
|
||||
# We return a backend with *something* that's not nil that we can check for.
|
||||
var
|
||||
key = VerifyingKey(icLen: 123)
|
||||
vkpPtr: ptr VerifyingKey = key.addr
|
||||
return CircomCompat(vkp: vkpPtr)
|
||||
|
||||
suite "Test BackendFactory":
|
||||
let
|
||||
utilsMock = BackendUtilsMock()
|
||||
circuitDir = "testecircuitdir"
|
||||
|
||||
setup:
|
||||
createDir(circuitDir)
|
||||
|
||||
teardown:
|
||||
removeDir(circuitDir)
|
||||
|
||||
test "Should create backend from cli config":
|
||||
let
|
||||
config = CodexConf(
|
||||
cmd: StartUpCmd.persistence,
|
||||
nat: ValidIpAddress.init("127.0.0.1"),
|
||||
discoveryIp: ValidIpAddress.init(IPv4_any()),
|
||||
metricsAddress: ValidIpAddress.init("127.0.0.1"),
|
||||
persistenceCmd: PersistenceCmd.prover,
|
||||
marketplaceAddress: EthAddress.example.some,
|
||||
circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"),
|
||||
circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"),
|
||||
circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey")
|
||||
)
|
||||
backend = config.initializeBackend(utilsMock).tryGet
|
||||
|
||||
check:
|
||||
backend.vkp != nil
|
||||
utilsMock.argR1csFile == $config.circomR1cs
|
||||
utilsMock.argWasmFile == $config.circomWasm
|
||||
utilsMock.argZKeyFile == $config.circomZkey
|
||||
|
||||
test "Should create backend from local files":
|
||||
let
|
||||
config = CodexConf(
|
||||
cmd: StartUpCmd.persistence,
|
||||
nat: ValidIpAddress.init("127.0.0.1"),
|
||||
discoveryIp: ValidIpAddress.init(IPv4_any()),
|
||||
metricsAddress: ValidIpAddress.init("127.0.0.1"),
|
||||
persistenceCmd: PersistenceCmd.prover,
|
||||
marketplaceAddress: EthAddress.example.some,
|
||||
|
||||
# Set the circuitDir such that the tests/circuits/fixtures/ files
|
||||
# will be picked up as local files:
|
||||
circuitDir: OutDir("tests/circuits/fixtures")
|
||||
)
|
||||
backend = config.initializeBackend(utilsMock).tryGet
|
||||
|
||||
check:
|
||||
backend.vkp != nil
|
||||
utilsMock.argR1csFile == config.circuitDir / "proof_main.r1cs"
|
||||
utilsMock.argWasmFile == config.circuitDir / "proof_main.wasm"
|
||||
utilsMock.argZKeyFile == config.circuitDir / "proof_main.zkey"
|
||||
|
||||
test "Should suggest usage of downloader tool when files not available":
|
||||
let
|
||||
config = CodexConf(
|
||||
cmd: StartUpCmd.persistence,
|
||||
nat: ValidIpAddress.init("127.0.0.1"),
|
||||
discoveryIp: ValidIpAddress.init(IPv4_any()),
|
||||
metricsAddress: ValidIpAddress.init("127.0.0.1"),
|
||||
persistenceCmd: PersistenceCmd.prover,
|
||||
marketplaceAddress: EthAddress.example.some,
|
||||
circuitDir: OutDir(circuitDir)
|
||||
)
|
||||
backendResult = config.initializeBackend(utilsMock)
|
||||
|
||||
check:
|
||||
backendResult.isErr
|
||||
@ -15,6 +15,8 @@ import pkg/codex/chunker
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/slots
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/conf
|
||||
import pkg/confutils/defs
|
||||
import pkg/poseidon2/io
|
||||
import pkg/codex/utils/poseidon2digest
|
||||
|
||||
@ -24,38 +26,36 @@ import ./backends/helpers
|
||||
|
||||
suite "Test Prover":
|
||||
let
|
||||
slotId = 1
|
||||
samples = 5
|
||||
ecK = 3
|
||||
ecM = 2
|
||||
numDatasetBlocks = 8
|
||||
blockSize = DefaultBlockSize
|
||||
cellSize = DefaultCellSize
|
||||
repoTmp = TempLevelDb.new()
|
||||
metaTmp = TempLevelDb.new()
|
||||
challenge = 1234567.toF.toBytes.toArray32
|
||||
|
||||
var
|
||||
datasetBlocks: seq[bt.Block]
|
||||
store: BlockStore
|
||||
manifest: Manifest
|
||||
protected: Manifest
|
||||
verifiable: Manifest
|
||||
sampler: Poseidon2Sampler
|
||||
prover: Prover
|
||||
|
||||
setup:
|
||||
let
|
||||
repoDs = repoTmp.newDb()
|
||||
metaDs = metaTmp.newDb()
|
||||
config = CodexConf(
|
||||
cmd: StartUpCmd.persistence,
|
||||
nat: ValidIpAddress.init("127.0.0.1"),
|
||||
discoveryIp: ValidIpAddress.init(IPv4_any()),
|
||||
metricsAddress: ValidIpAddress.init("127.0.0.1"),
|
||||
persistenceCmd: PersistenceCmd.prover,
|
||||
circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"),
|
||||
circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"),
|
||||
circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"),
|
||||
numProofSamples: samples
|
||||
)
|
||||
backend = config.initializeBackend().tryGet()
|
||||
|
||||
store = RepoStore.new(repoDs, metaDs)
|
||||
|
||||
(manifest, protected, verifiable) =
|
||||
await createVerifiableManifest(
|
||||
store,
|
||||
numDatasetBlocks,
|
||||
ecK, ecM,
|
||||
blockSize,
|
||||
cellSize)
|
||||
prover = Prover.new(store, backend, config.numProofSamples)
|
||||
|
||||
teardown:
|
||||
await repoTmp.destroyDb()
|
||||
@ -63,13 +63,41 @@ suite "Test Prover":
|
||||
|
||||
test "Should sample and prove a slot":
|
||||
let
|
||||
r1cs = "tests/circuits/fixtures/proof_main.r1cs"
|
||||
wasm = "tests/circuits/fixtures/proof_main.wasm"
|
||||
(_, _, verifiable) =
|
||||
await createVerifiableManifest(
|
||||
store,
|
||||
8, # number of blocks in the original dataset (before EC)
|
||||
5, # ecK
|
||||
3, # ecM
|
||||
blockSize,
|
||||
cellSize)
|
||||
|
||||
circomBackend = CircomCompat.init(r1cs, wasm)
|
||||
prover = Prover.new(store, circomBackend, samples)
|
||||
challenge = 1234567.toF.toBytes.toArray32
|
||||
(inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet
|
||||
let
|
||||
(inputs, proof) = (
|
||||
await prover.prove(1, verifiable, challenge)).tryGet
|
||||
|
||||
check:
|
||||
(await prover.verify(proof, inputs)).tryGet == true
|
||||
|
||||
test "Should generate valid proofs when slots consist of single blocks":
|
||||
|
||||
# To get single-block slots, we just need to set the number of blocks in
|
||||
# the original dataset to be the same as ecK. The total number of blocks
|
||||
# after generating random data for parity will be ecK + ecM, which will
|
||||
# match the number of slots.
|
||||
let
|
||||
(_, _, verifiable) =
|
||||
await createVerifiableManifest(
|
||||
store,
|
||||
2, # number of blocks in the original dataset (before EC)
|
||||
2, # ecK
|
||||
1, # ecM
|
||||
blockSize,
|
||||
cellSize)
|
||||
|
||||
let
|
||||
(inputs, proof) = (
|
||||
await prover.prove(1, verifiable, challenge)).tryGet
|
||||
|
||||
check:
|
||||
(await prover.verify(proof, inputs)).tryGet == true
|
||||
|
||||
@ -15,6 +15,7 @@ import pkg/codex/utils
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
import ../examples
|
||||
|
||||
type
|
||||
StoreProvider* = proc(): BlockStore {.gcsafe.}
|
||||
@ -56,6 +57,16 @@ proc commonBlockStoreTests*(name: string,
|
||||
(await store.putBlock(newBlock1)).tryGet()
|
||||
check (await store.hasBlock(newBlock1.cid)).tryGet()
|
||||
|
||||
test "putBlock raises onBlockStored":
|
||||
var storedCid = Cid.example
|
||||
proc onStored(cid: Cid) {.async.} =
|
||||
storedCid = cid
|
||||
store.onBlockStored = onStored.some()
|
||||
|
||||
(await store.putBlock(newBlock1)).tryGet()
|
||||
|
||||
check storedCid == newBlock1.cid
|
||||
|
||||
test "getBlock":
|
||||
(await store.putBlock(newBlock)).tryGet()
|
||||
let blk = await store.getBlock(newBlock.cid)
|
||||
|
||||
71
tests/codex/stores/repostore/testcoders.nim
Normal file
71
tests/codex/stores/repostore/testcoders.nim
Normal file
@ -0,0 +1,71 @@
|
||||
import std/unittest
|
||||
import std/random
|
||||
|
||||
import pkg/stew/objects
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import pkg/codex/clock
|
||||
import pkg/codex/stores/repostore/types
|
||||
import pkg/codex/stores/repostore/coders
|
||||
|
||||
import ../../helpers
|
||||
|
||||
checksuite "Test coders":
|
||||
|
||||
proc rand(T: type NBytes): T =
|
||||
rand(Natural).NBytes
|
||||
|
||||
proc rand(E: type[enum]): E =
|
||||
let ordinals = enumRangeInt64(E)
|
||||
E(ordinals[rand(ordinals.len - 1)])
|
||||
|
||||
proc rand(T: type QuotaUsage): T =
|
||||
QuotaUsage(
|
||||
used: rand(NBytes),
|
||||
reserved: rand(NBytes)
|
||||
)
|
||||
|
||||
proc rand(T: type BlockMetadata): T =
|
||||
BlockMetadata(
|
||||
expiry: rand(SecondsSince1970),
|
||||
size: rand(NBytes),
|
||||
refCount: rand(Natural)
|
||||
)
|
||||
|
||||
proc rand(T: type DeleteResult): T =
|
||||
DeleteResult(
|
||||
kind: rand(DeleteResultKind),
|
||||
released: rand(NBytes)
|
||||
)
|
||||
|
||||
proc rand(T: type StoreResult): T =
|
||||
StoreResult(
|
||||
kind: rand(StoreResultKind),
|
||||
used: rand(NBytes)
|
||||
)
|
||||
|
||||
test "Natural encode/decode":
|
||||
for val in newSeqWith[Natural](100, rand(Natural)) & @[Natural.low, Natural.high]:
|
||||
check:
|
||||
success(val) == Natural.decode(encode(val))
|
||||
|
||||
test "QuotaUsage encode/decode":
|
||||
for val in newSeqWith[QuotaUsage](100, rand(QuotaUsage)):
|
||||
check:
|
||||
success(val) == QuotaUsage.decode(encode(val))
|
||||
|
||||
test "BlockMetadata encode/decode":
|
||||
for val in newSeqWith[BlockMetadata](100, rand(BlockMetadata)):
|
||||
check:
|
||||
success(val) == BlockMetadata.decode(encode(val))
|
||||
|
||||
test "DeleteResult encode/decode":
|
||||
for val in newSeqWith[DeleteResult](100, rand(DeleteResult)):
|
||||
check:
|
||||
success(val) == DeleteResult.decode(encode(val))
|
||||
|
||||
test "StoreResult encode/decode":
|
||||
for val in newSeqWith[StoreResult](100, rand(StoreResult)):
|
||||
check:
|
||||
success(val) == StoreResult.decode(encode(val))
|
||||
@ -34,10 +34,10 @@ checksuite "BlockMaintainer":
|
||||
var testBe2: BlockExpiration
|
||||
var testBe3: BlockExpiration
|
||||
|
||||
proc createTestExpiration(expiration: SecondsSince1970): BlockExpiration =
|
||||
proc createTestExpiration(expiry: SecondsSince1970): BlockExpiration =
|
||||
BlockExpiration(
|
||||
cid: bt.Block.example.cid,
|
||||
expiration: expiration
|
||||
expiry: expiry
|
||||
)
|
||||
|
||||
setup:
|
||||
@ -186,4 +186,3 @@ checksuite "BlockMaintainer":
|
||||
await invokeTimerManyTimes()
|
||||
# Second new block has expired
|
||||
check mockRepoStore.delBlockCids == [testBe1.cid, testBe2.cid, testBe3.cid, testBe4.cid, testBe5.cid]
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user