Compare commits
227
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
96510df5cf | ||
|
|
07c3e5fc18 | ||
|
|
d6cd7762c4 | ||
|
|
c5de3dcb4c | ||
|
|
37b98dbb05 | ||
|
|
288ad81faa | ||
|
|
64e917f3c2 | ||
|
|
f33e343432 | ||
|
|
855b627823 | ||
|
|
f58a42a1ca | ||
|
|
789852bf1a | ||
|
|
5c56e4796f | ||
|
|
5dacffa720 | ||
|
|
9629060570 | ||
|
|
7e37299deb | ||
|
|
eaa69bcc95 | ||
|
|
df5efa4dde | ||
|
|
320865460d | ||
|
|
a96a9ce319 | ||
|
|
3ab1e2ea47 | ||
|
|
9b432ded46 | ||
|
|
e862094014 | ||
|
|
d701e2a73d | ||
|
|
22e37c15e5 | ||
|
|
f7de11e9c2 | ||
|
|
278b6c95c2 | ||
|
|
fe75609ecb | ||
|
|
3af71b1f66 | ||
|
|
3cd989c04a | ||
|
|
804423ec81 | ||
|
|
92a5a1e361 | ||
|
|
0c961b8348 | ||
|
|
dc7a034566 | ||
|
|
3459547f20 | ||
|
|
5d9247f82f | ||
|
|
9e80b650c5 | ||
|
|
ce6cbf59d2 | ||
|
|
23fdc2d60a | ||
|
|
fee42c785e | ||
|
|
02d1292acc | ||
|
|
1e035d6a68 | ||
|
|
413a46c761 | ||
|
|
c1e3e8be09 | ||
|
|
0784804fa4 | ||
|
|
8969e9a080 | ||
|
|
a62c6f5108 | ||
|
|
e24d5e02f2 | ||
|
|
50e58dc799 | ||
|
|
b1197975a1 | ||
|
|
e4512438ed | ||
|
|
2c549c5410 | ||
|
|
d6afc6874b | ||
|
|
940b8122e7 | ||
|
|
2586f4a756 | ||
|
|
951da1888d | ||
|
|
caa0423bd5 | ||
|
|
064fc241cb | ||
|
|
1a8b7b79ef | ||
|
|
9a227b3d0e | ||
|
|
c73fa186fc | ||
|
|
4a151880d4 | ||
|
|
48f4614e5d | ||
|
|
ec644eed4a | ||
|
|
0c7b8fb33e | ||
|
|
0aac2d8e98 | ||
|
|
ee0193c879 | ||
|
|
e45ed0c21e | ||
|
|
2292d2e672 | ||
|
|
8ef190baad | ||
|
|
9461968bd3 | ||
|
|
01cefcc008 | ||
|
|
d13f6c94d3 | ||
|
|
4801e33986 | ||
|
|
5b42b764fb | ||
|
|
2afcb92d08 | ||
|
|
a4ba1fbe2c | ||
|
|
8ee5eb5767 | ||
|
|
70622cf923 | ||
|
|
459cb2e981 | ||
|
|
95aa6fd4b2 | ||
|
|
1d9f22c96d | ||
|
|
c24861e932 | ||
|
|
8274c4da14 | ||
|
|
e45b8bde54 | ||
|
|
1af216592a | ||
|
|
37f037c7b0 | ||
|
|
e19311cef9 | ||
|
|
b8476f697e | ||
|
|
3e2cad3c17 | ||
|
|
770ba6dcdc | ||
|
|
5cecb63307 | ||
|
|
33153a2b76 | ||
|
|
ac8f01c922 | ||
|
|
4ae17c5b1d | ||
|
|
35e9dd5894 | ||
|
|
abf297263c | ||
|
|
ed764bc38c | ||
|
|
047384a7af | ||
|
|
03c6b287cf | ||
|
|
d7c3fe6c5f | ||
|
|
86d5ab22f6 | ||
|
|
4bcdbf3e8c | ||
|
|
f2442fc032 | ||
|
|
694b16fdd6 | ||
|
|
10b01362e0 | ||
|
|
7989cdd1a2 | ||
|
|
58ed7ad41d | ||
|
|
8d5f68609a | ||
|
|
b5fd8954ba | ||
|
|
f84bba801b | ||
|
|
513853d929 | ||
|
|
a40d5d77d7 | ||
|
|
23622245f0 | ||
|
|
f2b84ebfd6 | ||
|
|
9853a0b7db | ||
|
|
025c85c1aa | ||
|
|
f0cbc0a53a | ||
|
|
e91a574b2c | ||
|
|
f18ff24bb6 | ||
|
|
529d48a758 | ||
|
|
d136345df4 | ||
|
|
a3b9e7bf8d | ||
|
|
6f778ec04f | ||
|
|
4b7ceda572 | ||
|
|
797dc096da | ||
|
|
208cd2e8dc | ||
|
|
db46a0c686 | ||
|
|
c2df15436f | ||
|
|
dfa2322127 | ||
|
|
92d504ef9c | ||
|
|
4990bb2282 | ||
|
|
d032b77abe | ||
|
|
e7a59de207 | ||
|
|
fd3567347b | ||
|
|
f67e67c493 | ||
|
|
339cf2b824 | ||
|
|
e743a1cd7b | ||
|
|
7c4ad416d4 | ||
|
|
cfb6297357 | ||
|
|
f1f3b0f173 | ||
|
|
704847001d | ||
|
|
f4c622a1d3 | ||
|
|
3c7892e4be | ||
|
|
833421b2b2 | ||
|
|
55cc0ab0ef | ||
|
|
0129af6fd7 | ||
|
|
d48caa44d6 | ||
|
|
e0755a1101 | ||
|
|
1dd17037ba | ||
|
|
22e6439731 | ||
|
|
693880069b | ||
|
|
365032978b | ||
|
|
eb2e61938e | ||
|
|
5c1ffbb8af | ||
|
|
3e12baaafe | ||
|
|
605bb6411f | ||
|
|
2554645abc | ||
|
|
2dfcf20ecd | ||
|
|
c35784c90f | ||
|
|
b54c9ff9a3 | ||
|
|
ff4711e802 | ||
|
|
b8d6ac929b | ||
|
|
acb0bf4f29 | ||
|
|
8fe0bd6307 | ||
|
|
e6a5838b05 | ||
|
|
5c65d1d74e | ||
|
|
292b4b9b06 | ||
|
|
ddbe5b111a | ||
|
|
a0abea4432 | ||
|
|
2f39327db2 | ||
|
|
d452293cac | ||
|
|
3e245b707c | ||
|
|
9e842207ab | ||
|
|
e29ffe4f9c | ||
|
|
d8a6df5845 | ||
|
|
1859994ec6 | ||
|
|
60b489ced1 | ||
|
|
b72b4a850b | ||
|
|
fc942b11f8 | ||
|
|
2c88ddfb6b | ||
|
|
e352e5c65c | ||
|
|
2cbe030cff | ||
|
|
a38e93a607 | ||
|
|
269365e101 | ||
|
|
02ca9db001 | ||
|
|
672092b232 | ||
|
|
1bd84a4892 | ||
|
|
5896735884 | ||
|
|
bebeb3766e | ||
|
|
e0cbf8c84d | ||
|
|
3ed91d7310 | ||
|
|
9573814574 | ||
|
|
5313f8a7ac | ||
|
|
c239d555dc | ||
|
|
2e9d7641a3 | ||
|
|
fc9249da20 | ||
|
|
978e085219 | ||
|
|
960b0c3788 | ||
|
|
9ca4bf8afc | ||
|
|
cfdc25335c | ||
|
|
04f087efe4 | ||
|
|
f6aa122245 | ||
|
|
db4c4a87e0 | ||
|
|
ffb5eb294a | ||
|
|
6a8c74e02a | ||
|
|
200de1d7f7 | ||
|
|
f801cb082e | ||
|
|
2a61dad556 | ||
|
|
2d90349b7b | ||
|
|
c4b6d01530 | ||
|
|
c9fedac592 | ||
|
|
cedec0d4cc | ||
|
|
769b9c3aca | ||
|
|
88c675adf9 | ||
|
|
75fcc68caf | ||
|
|
a41272f160 | ||
|
|
8e018cbae9 | ||
|
|
3c447eb4c5 | ||
|
|
d53b760731 | ||
|
|
fcadceb009 | ||
|
|
a02d9558e5 | ||
|
|
b3013a9b65 | ||
|
|
6b0a16b627 | ||
|
|
eac06e8b3a | ||
|
|
f7fa35c7ba | ||
|
|
a7526aaed1 | ||
|
|
e7d9e833f1 |
@@ -0,0 +1,10 @@
|
||||
# Set default behavior to automatically normalize line endings.
|
||||
* text=auto
|
||||
|
||||
# Force bash scripts to always use lf line endings so that if a repo is accessed
|
||||
# in Unix via a file share from Windows, the scripts will work.
|
||||
*.sh text eol=lf
|
||||
|
||||
# Likewise, force cmd and batch scripts to always use crlf
|
||||
*.cmd text eol=crlf
|
||||
*.bat text eol=crlf
|
||||
@@ -1,5 +1,6 @@
|
||||
name: Docker - AutoClient
|
||||
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
@@ -12,15 +13,15 @@ on:
|
||||
- 'Framework/**'
|
||||
- 'ProjectPlugins/**'
|
||||
- .github/workflows/docker-autoclient.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: Tools/AutoClient/docker/Dockerfile
|
||||
docker_repo: codexstorage/codex-autoclient
|
||||
dockerhub_repo: codexstorage/codex-autoclient
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
|
||||
@@ -13,16 +13,15 @@ on:
|
||||
- 'Framework/**'
|
||||
- 'ProjectPlugins/**'
|
||||
- .github/workflows/docker-discordbot.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: Tools/BiblioTech/docker/Dockerfile
|
||||
docker_repo: codexstorage/codex-discordbot
|
||||
dockerhub_repo: codexstorage/codex-discordbot
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
|
||||
@@ -11,17 +11,16 @@ on:
|
||||
- 'Tools/KeyMaker/**'
|
||||
- 'Framework/**'
|
||||
- 'ProjectPlugins/**'
|
||||
- .github/workflows/docker-KeyMaker.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
- .github/workflows/docker-keymaker.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: Tools/KeyMaker/docker/Dockerfile
|
||||
docker_repo: codexstorage/codex-keymaker
|
||||
dockerhub_repo: codexstorage/codex-keymaker
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
|
||||
@@ -12,16 +12,15 @@ on:
|
||||
- 'Framework/**'
|
||||
- 'ProjectPlugins/**'
|
||||
- .github/workflows/docker-marketinsights.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: Tools/MarketInsights/Dockerfile
|
||||
docker_repo: codexstorage/codex-marketinsights
|
||||
dockerhub_repo: codexstorage/codex-marketinsights
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
name: Reusable - Docker
|
||||
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_file:
|
||||
default: docker/Dockerfile
|
||||
description: Dockerfile
|
||||
required: false
|
||||
type: string
|
||||
docker_repo:
|
||||
default: codexstorage/cs-codex-dist-tests
|
||||
description: DockerHub repository
|
||||
required: false
|
||||
type: string
|
||||
tag_latest:
|
||||
default: true
|
||||
description: Set latest tag for Docker images
|
||||
required: false
|
||||
type: boolean
|
||||
tag_sha:
|
||||
default: true
|
||||
description: Set Git short commit as Docker tag
|
||||
required: false
|
||||
type: boolean
|
||||
tag_suffix:
|
||||
default: ''
|
||||
description: Suffix for Docker images tag
|
||||
required: false
|
||||
type: string
|
||||
|
||||
|
||||
env:
|
||||
DOCKER_FILE: ${{ inputs.docker_file }}
|
||||
DOCKER_REPO: ${{ inputs.docker_repo }}
|
||||
TAG_LATEST: ${{ inputs.tag_latest }}
|
||||
TAG_SHA: ${{ inputs.tag_sha }}
|
||||
TAG_SUFFIX: ${{ inputs.tag_suffix }}
|
||||
|
||||
|
||||
jobs:
|
||||
# Build platform specific image
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
target:
|
||||
- os: linux
|
||||
arch: amd64
|
||||
- os: linux
|
||||
arch: arm64
|
||||
include:
|
||||
- target:
|
||||
os: linux
|
||||
arch: amd64
|
||||
builder: ubuntu-22.04
|
||||
- target:
|
||||
os: linux
|
||||
arch: arm64
|
||||
builder: buildjet-4vcpu-ubuntu-2204-arm
|
||||
|
||||
name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
|
||||
runs-on: ${{ matrix.builder }}
|
||||
env:
|
||||
PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
|
||||
- name: Docker - Set up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Build and Push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ${{ env.DOCKER_FILE }}
|
||||
platforms: ${{ env.PLATFORM }}
|
||||
push: true
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Docker - Export digest
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Docker - Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ matrix.target.arch }}
|
||||
path: /tmp/digests
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
|
||||
# Publish multi-platform image
|
||||
publish:
|
||||
name: Publish multi-platform image
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Docker - Variables
|
||||
run: |
|
||||
# Adjust custom suffix when set and
|
||||
if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
|
||||
echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
|
||||
fi
|
||||
# Disable SHA tags on tagged release
|
||||
if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
|
||||
echo "TAG_SHA=false" >>$GITHUB_ENV
|
||||
fi
|
||||
# Handle latest and latest-custom using raw
|
||||
if [[ ${{ env.TAG_SHA }} == "false" ]]; then
|
||||
echo "TAG_LATEST=false" >>$GITHUB_ENV
|
||||
echo "TAG_RAW=true" >>$GITHUB_ENV
|
||||
if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
|
||||
echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
|
||||
else
|
||||
echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
|
||||
fi
|
||||
else
|
||||
echo "TAG_RAW=false" >>$GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Docker - Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Docker - Set up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
flavor: |
|
||||
latest=${{ env.TAG_LATEST }}
|
||||
suffix=${{ env.TAG_SUFFIX }},onlatest=true
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=raw,enable=${{ env.TAG_RAW }},value=latest
|
||||
type=sha,enable=${{ env.TAG_SHA }}
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
|
||||
|
||||
- name: Docker - Inspect image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}
|
||||
@@ -1,5 +1,6 @@
|
||||
name: Docker - Rewarder Bot
|
||||
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
@@ -12,15 +13,15 @@ on:
|
||||
- 'Framework/**'
|
||||
- 'ProjectPlugins/**'
|
||||
- .github/workflows/docker-rewarder.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: Tools/TestNetRewarder/docker/Dockerfile
|
||||
docker_repo: codexstorage/codex-rewarderbot
|
||||
dockerhub_repo: codexstorage/codex-rewarderbot
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
|
||||
@@ -11,12 +11,15 @@ on:
|
||||
- docker/Dockerfile
|
||||
- docker/docker-entrypoint.sh
|
||||
- .github/workflows/docker-runner.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: docker/Dockerfile
|
||||
dockerhub_repo: codexstorage/cs-codex-dist-tests
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
@@ -80,7 +80,7 @@ env:
|
||||
TESTS_TARGET_DURATION: 2d
|
||||
TESTS_FILTER: ""
|
||||
TESTS_CLEANUP: true
|
||||
JOB_MANIFEST: docker/continuous-tests-job.yaml
|
||||
JOB_MANIFEST: docker/job-continuous-tests.yaml
|
||||
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
|
||||
KUBE_VERSION: v1.28.2
|
||||
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if: false
|
||||
run: |
|
||||
# Variables
|
||||
# We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node
|
||||
# We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
|
||||
duration=600
|
||||
namespace="${{ env.NAMESPACE }}"
|
||||
pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
|
||||
|
||||
@@ -32,7 +32,7 @@ env:
|
||||
NAMEPREFIX: d-tests-runner
|
||||
NAMESPACE: default
|
||||
COMMAND: dotnet test Tests/CodexTests
|
||||
JOB_MANIFEST: docker/dist-tests-job.yaml
|
||||
JOB_MANIFEST: docker/job-dist-tests.yaml
|
||||
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
|
||||
KUBE_VERSION: v1.28.2
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
name: Run Release Tests
|
||||
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
codexdockerimage:
|
||||
description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
|
||||
required: true
|
||||
type: string
|
||||
workflow_call:
|
||||
inputs:
|
||||
source:
|
||||
description: Repository with tests (current)
|
||||
required: false
|
||||
type: string
|
||||
branch:
|
||||
description: Branch with tests (master)
|
||||
required: false
|
||||
type: string
|
||||
codexdockerimage:
|
||||
description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
|
||||
required: true
|
||||
type: string
|
||||
workflow_source:
|
||||
description: Workflow source
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
env:
|
||||
SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
|
||||
TEST_TYPE: release-tests
|
||||
NAMEPREFIX: r-tests
|
||||
NAMESPACE: default
|
||||
JOB_MANIFEST: docker/job-release-tests.yaml
|
||||
COMMAND: dotnet test Tests/CodexReleaseTests
|
||||
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
|
||||
KUBE_VERSION: v1.30.5
|
||||
|
||||
|
||||
jobs:
|
||||
run_tests:
|
||||
name: Run Release Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ inputs.workflow_source }}
|
||||
|
||||
- name: Variables
|
||||
run: |
|
||||
RUNID=$(date +%Y%m%d-%H%M%S)
|
||||
echo "RUNID=${RUNID}" >> $GITHUB_ENV
|
||||
echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
|
||||
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
|
||||
[[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
|
||||
[[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
|
||||
[[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
|
||||
echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
|
||||
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
|
||||
[[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
|
||||
echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
|
||||
|
||||
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
|
||||
uses: azure/setup-kubectl@v4
|
||||
with:
|
||||
version: ${{ env.KUBE_VERSION }}
|
||||
|
||||
- name: Kubectl - Kubeconfig
|
||||
run: |
|
||||
mkdir -p "${HOME}"/.kube
|
||||
echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
|
||||
|
||||
- name: Kubectl - Create Job to run tests
|
||||
run: |
|
||||
envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
|
||||
|
||||
- name: Tests Identification
|
||||
run: |
|
||||
echo "----"
|
||||
echo "Repository: ${{ env.SOURCE }}"
|
||||
echo "Branch: ${{ env.BRANCH }}"
|
||||
echo "Runner job: ${{ env.NAMEPREFIX }}"
|
||||
echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
|
||||
echo "Runner namespace: ${{ env.NAMESPACE }}"
|
||||
echo "----"
|
||||
|
||||
- name: Show Runner logs
|
||||
run: |
|
||||
# Variables
|
||||
# We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node
|
||||
namespace="${{ env.NAMESPACE }}"
|
||||
pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
|
||||
|
||||
# Check Pod status
|
||||
WAIT=120
|
||||
SECONDS=0
|
||||
sleep=1
|
||||
while (( SECONDS < WAIT )); do
|
||||
phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
|
||||
[[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
|
||||
sleep $sleep
|
||||
done
|
||||
|
||||
# Get logs
|
||||
while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
|
||||
echo "Show ${pod} logs ..."
|
||||
echo "----"
|
||||
kubectl logs $pod -n $namespace -f || true
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Set workflow status from job status
|
||||
run: |
|
||||
sleep 5
|
||||
job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
|
||||
echo "Job status: $job_status"
|
||||
if [[ "${job_status}" != "Complete" ]]; then exit 1; fi
|
||||
@@ -3,3 +3,4 @@ obj
|
||||
bin
|
||||
.vscode
|
||||
Tools/AutoClient/datapath
|
||||
.editorconfig
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
namespace NethereumWorkflow.BlockUtils
|
||||
namespace BlockchainUtils
|
||||
{
|
||||
public class BlockCache
|
||||
{
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
namespace NethereumWorkflow.BlockUtils
|
||||
namespace BlockchainUtils
|
||||
{
|
||||
public class BlockTimeEntry
|
||||
{
|
||||
+5
-3
@@ -1,6 +1,6 @@
|
||||
using Logging;
|
||||
|
||||
namespace NethereumWorkflow.BlockUtils
|
||||
namespace BlockchainUtils
|
||||
{
|
||||
public class BlockTimeFinder
|
||||
{
|
||||
@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
|
||||
public ulong? GetHighestBlockNumberBefore(DateTime moment)
|
||||
{
|
||||
bounds.Initialize();
|
||||
if (moment <= bounds.Genesis.Utc) return null;
|
||||
if (moment < bounds.Genesis.Utc) return null;
|
||||
if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
|
||||
if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
|
||||
|
||||
return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
|
||||
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
|
||||
public ulong? GetLowestBlockNumberAfter(DateTime moment)
|
||||
{
|
||||
bounds.Initialize();
|
||||
if (moment >= bounds.Current.Utc) return null;
|
||||
if (moment > bounds.Current.Utc) return null;
|
||||
if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
|
||||
if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
|
||||
|
||||
return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;
|
||||
+7
-1
@@ -1,5 +1,11 @@
|
||||
namespace NethereumWorkflow.BlockUtils
|
||||
namespace BlockchainUtils
|
||||
{
|
||||
public interface IWeb3Blocks
|
||||
{
|
||||
ulong GetCurrentBlockNumber();
|
||||
DateTime? GetTimestampForBlock(ulong blockNumber);
|
||||
}
|
||||
|
||||
public class BlockchainBounds
|
||||
{
|
||||
private readonly BlockCache cache;
|
||||
@@ -0,0 +1,16 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Logging\Logging.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
using Nethereum.Hex.HexTypes;
|
||||
using System.Numerics;
|
||||
|
||||
namespace NethereumWorkflow
|
||||
namespace BlockchainUtils
|
||||
{
|
||||
public static class ConversionExtensions
|
||||
{
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
@@ -9,6 +9,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\FileUtils\FileUtils.csproj" />
|
||||
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
|
||||
<ProjectReference Include="..\WebUtils\WebUtils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using KubernetesWorkflow;
|
||||
using KubernetesWorkflow.Types;
|
||||
using Logging;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using WebUtils;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
@@ -8,16 +9,16 @@ namespace Core
|
||||
private readonly IToolsFactory toolsFactory;
|
||||
private readonly PluginManager manager = new PluginManager();
|
||||
|
||||
public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
|
||||
public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
|
||||
{
|
||||
toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
|
||||
toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
|
||||
|
||||
Tools = toolsFactory.CreateTools();
|
||||
manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
|
||||
}
|
||||
|
||||
public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
|
||||
: this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
|
||||
: this(log, configuration, fileManagerRootFolder, new DefaultWebCallTimeSet(), new DefaultK8sTimeSet())
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
using FileUtils;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using WebUtils;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
|
||||
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactory, IFileTool
|
||||
{
|
||||
ITimeSet TimeSet { get; }
|
||||
IWebCallTimeSet WebCallTimeSet { get; }
|
||||
IK8sTimeSet K8STimeSet { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Deletes kubernetes and tracked file resources.
|
||||
@@ -25,13 +27,6 @@ namespace Core
|
||||
ILog GetLog();
|
||||
}
|
||||
|
||||
public interface IHttpFactoryTool
|
||||
{
|
||||
IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
|
||||
IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
|
||||
IHttp CreateHttp(string id);
|
||||
}
|
||||
|
||||
public interface IFileTool
|
||||
{
|
||||
IFileManager GetFileManager();
|
||||
@@ -40,18 +35,22 @@ namespace Core
|
||||
internal class PluginTools : IPluginTools
|
||||
{
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
private readonly HttpFactory httpFactory;
|
||||
private readonly IFileManager fileManager;
|
||||
private readonly LogPrefixer log;
|
||||
|
||||
internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
|
||||
internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
|
||||
{
|
||||
this.log = new LogPrefixer(log);
|
||||
this.workflowCreator = workflowCreator;
|
||||
TimeSet = timeSet;
|
||||
httpFactory = new HttpFactory(log, webCallTimeSet);
|
||||
WebCallTimeSet = webCallTimeSet;
|
||||
K8STimeSet = k8STimeSet;
|
||||
fileManager = new FileManager(log, fileManagerRootFolder);
|
||||
}
|
||||
|
||||
public ITimeSet TimeSet { get; }
|
||||
public IWebCallTimeSet WebCallTimeSet { get; }
|
||||
public IK8sTimeSet K8STimeSet { get; }
|
||||
|
||||
public void ApplyLogPrefix(string prefix)
|
||||
{
|
||||
@@ -60,17 +59,17 @@ namespace Core
|
||||
|
||||
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
|
||||
{
|
||||
return CreateHttp(id, onClientCreated, TimeSet);
|
||||
return httpFactory.CreateHttp(id, onClientCreated);
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
|
||||
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet)
|
||||
{
|
||||
return new Http(id, log, ts, onClientCreated);
|
||||
return httpFactory.CreateHttp(id, onClientCreated, timeSet);
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(string id)
|
||||
{
|
||||
return new Http(id, log, TimeSet);
|
||||
return httpFactory.CreateHttp(id);
|
||||
}
|
||||
|
||||
public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using WebUtils;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
@@ -13,19 +14,21 @@ namespace Core
|
||||
private readonly ILog log;
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
private readonly string fileManagerRootFolder;
|
||||
private readonly ITimeSet timeSet;
|
||||
private readonly IWebCallTimeSet webCallTimeSet;
|
||||
private readonly IK8sTimeSet k8STimeSet;
|
||||
|
||||
public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
|
||||
public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
|
||||
{
|
||||
this.log = log;
|
||||
workflowCreator = new WorkflowCreator(log, configuration);
|
||||
this.fileManagerRootFolder = fileManagerRootFolder;
|
||||
this.timeSet = timeSet;
|
||||
this.webCallTimeSet = webCallTimeSet;
|
||||
this.k8STimeSet = k8STimeSet;
|
||||
}
|
||||
|
||||
public PluginTools CreateTools()
|
||||
{
|
||||
return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
|
||||
return new PluginTools(log, workflowCreator, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
public class GiveRewardsCommand
|
||||
{
|
||||
public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
|
||||
public string[] EventsOverview { get; set; } = Array.Empty<string>();
|
||||
public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
|
||||
public string[] Errors { get; set; } = Array.Empty<string>();
|
||||
|
||||
public bool HasAny()
|
||||
{
|
||||
@@ -16,4 +17,10 @@
|
||||
public ulong RewardId { get; set; }
|
||||
public string[] UserAddresses { get; set; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
public class ChainEventMessage
|
||||
{
|
||||
public ulong BlockNumber { get; set; }
|
||||
public string Message { get; set; } = string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
@@ -14,6 +14,12 @@ namespace FileUtils
|
||||
Label = label;
|
||||
}
|
||||
|
||||
public static TrackedFile FromPath(ILog log, string filepath)
|
||||
{
|
||||
// todo: I don't wanne have to do this to call upload.
|
||||
return new TrackedFile(log, filepath, string.Empty);
|
||||
}
|
||||
|
||||
public string Filename { get; }
|
||||
public string Label { get; }
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using CodexContractsPlugin;
|
||||
using BlockchainUtils;
|
||||
using CodexContractsPlugin;
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using GethPlugin;
|
||||
using Logging;
|
||||
|
||||
@@ -18,18 +20,29 @@ namespace GethConnector
|
||||
return null;
|
||||
}
|
||||
|
||||
var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
|
||||
|
||||
var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
|
||||
|
||||
var contractsDeployment = new CodexContractsDeployment(
|
||||
config: config,
|
||||
marketplaceAddress: GethInput.MarketplaceAddress,
|
||||
abi: GethInput.ABI,
|
||||
tokenAddress: GethInput.TokenAddress
|
||||
);
|
||||
|
||||
var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
|
||||
var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
|
||||
|
||||
return new GethConnector(gethNode, contracts);
|
||||
}
|
||||
|
||||
private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
|
||||
{
|
||||
var func = new ConfigurationFunctionBase();
|
||||
var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
|
||||
return response.ReturnValue1;
|
||||
}
|
||||
|
||||
private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
|
||||
{
|
||||
GethNode = gethNode;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
@@ -45,8 +45,12 @@
|
||||
private static string? GetEnvVar(List<string> error, string name)
|
||||
{
|
||||
var result = Environment.GetEnvironmentVariable(name);
|
||||
if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
|
||||
return result;
|
||||
if (string.IsNullOrEmpty(result))
|
||||
{
|
||||
error.Add($"'{name}' is not set.");
|
||||
return null;
|
||||
}
|
||||
return result.Trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+6
-6
@@ -3,7 +3,7 @@ using Logging;
|
||||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
public class CrashWatcher
|
||||
public class ContainerCrashWatcher
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly KubernetesClientConfiguration config;
|
||||
@@ -15,7 +15,7 @@ namespace KubernetesWorkflow
|
||||
private Task? worker;
|
||||
private Exception? workerException;
|
||||
|
||||
public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
|
||||
public ContainerCrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
|
||||
{
|
||||
this.log = log;
|
||||
this.config = config;
|
||||
@@ -45,7 +45,7 @@ namespace KubernetesWorkflow
|
||||
if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
|
||||
}
|
||||
|
||||
public bool HasContainerCrashed()
|
||||
public bool HasCrashed()
|
||||
{
|
||||
using var client = new Kubernetes(config);
|
||||
var result = HasContainerBeenRestarted(client);
|
||||
@@ -70,13 +70,13 @@ namespace KubernetesWorkflow
|
||||
using var client = new Kubernetes(config);
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
|
||||
|
||||
if (HasContainerBeenRestarted(client))
|
||||
{
|
||||
DownloadCrashedContainerLogs(client);
|
||||
return;
|
||||
}
|
||||
|
||||
token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ namespace KubernetesWorkflow
|
||||
private void DownloadCrashedContainerLogs(Kubernetes client)
|
||||
{
|
||||
using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
|
||||
var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
|
||||
var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
|
||||
handler.Log(stream);
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
using Logging;
|
||||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
public interface IDownloadedLog
|
||||
{
|
||||
string ContainerName { get; }
|
||||
|
||||
void IterateLines(Action<string> action, params string[] thatContain);
|
||||
string[] GetLinesContaining(string expectedString);
|
||||
string[] FindLinesThatContain(params string[] tags);
|
||||
string GetFilepath();
|
||||
void DeleteFile();
|
||||
}
|
||||
|
||||
internal class DownloadedLog : IDownloadedLog
|
||||
{
|
||||
private readonly LogFile logFile;
|
||||
|
||||
internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
|
||||
{
|
||||
logFile = logHandler.LogFile;
|
||||
ContainerName = containerName;
|
||||
}
|
||||
|
||||
public string ContainerName { get; }
|
||||
|
||||
public void IterateLines(Action<string> action, params string[] thatContain)
|
||||
{
|
||||
using var file = File.OpenRead(logFile.FullFilename);
|
||||
using var streamReader = new StreamReader(file);
|
||||
|
||||
var line = streamReader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
if (thatContain.All(line.Contains))
|
||||
{
|
||||
action(line);
|
||||
}
|
||||
line = streamReader.ReadLine();
|
||||
}
|
||||
}
|
||||
|
||||
public string[] GetLinesContaining(string expectedString)
|
||||
{
|
||||
using var file = File.OpenRead(logFile.FullFilename);
|
||||
using var streamReader = new StreamReader(file);
|
||||
var lines = new List<string>();
|
||||
|
||||
var line = streamReader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
if (line.Contains(expectedString))
|
||||
{
|
||||
lines.Add(line);
|
||||
}
|
||||
line = streamReader.ReadLine();
|
||||
}
|
||||
|
||||
return lines.ToArray(); ;
|
||||
}
|
||||
|
||||
public string[] FindLinesThatContain(params string[] tags)
|
||||
{
|
||||
var result = new List<string>();
|
||||
using var file = File.OpenRead(logFile.FullFilename);
|
||||
using var streamReader = new StreamReader(file);
|
||||
|
||||
var line = streamReader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
if (tags.All(line.Contains))
|
||||
{
|
||||
result.Add(line);
|
||||
}
|
||||
|
||||
line = streamReader.ReadLine();
|
||||
}
|
||||
|
||||
return result.ToArray();
|
||||
}
|
||||
|
||||
public string GetFilepath()
|
||||
{
|
||||
return logFile.FullFilename;
|
||||
}
|
||||
|
||||
public void DeleteFile()
|
||||
{
|
||||
File.Delete(logFile.FullFilename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,16 @@ namespace KubernetesWorkflow
|
||||
var all = client.Run(c => c.ListNamespace().Items);
|
||||
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
|
||||
|
||||
if (wait)
|
||||
{
|
||||
// If we're going to wait, trigger the delete for all the namespaces immediately.
|
||||
// Then wait for them to finish one by one.
|
||||
foreach (var ns in namespaces)
|
||||
{
|
||||
DeleteNamespace(ns, false);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var ns in namespaces)
|
||||
{
|
||||
DeleteNamespace(ns, wait);
|
||||
@@ -735,7 +745,10 @@ namespace KubernetesWorkflow
|
||||
throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
|
||||
$"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
|
||||
}
|
||||
return pods[0];
|
||||
var pod = pods[0];
|
||||
if (pod.Status == null) throw new Exception("Pod status unknown");
|
||||
if (string.IsNullOrEmpty(pod.Status.PodIP)) throw new Exception("Pod IP unknown");
|
||||
return pod;
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -906,7 +919,7 @@ namespace KubernetesWorkflow
|
||||
var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
|
||||
log.Error(msg);
|
||||
|
||||
DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
|
||||
DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
|
||||
|
||||
throw new Exception(msg);
|
||||
}
|
||||
@@ -946,13 +959,13 @@ namespace KubernetesWorkflow
|
||||
|
||||
#endregion
|
||||
|
||||
public CrashWatcher CreateCrashWatcher(RunningContainer container)
|
||||
public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
|
||||
{
|
||||
var containerName = container.Name;
|
||||
var podName = GetPodName(container);
|
||||
var recipeName = container.Recipe.Name;
|
||||
|
||||
return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
|
||||
return new ContainerCrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
|
||||
}
|
||||
|
||||
private V1Pod[] FindPodsByLabel(string podLabel)
|
||||
|
||||
@@ -24,8 +24,8 @@
|
||||
.Replace("]", "-")
|
||||
.Replace(",", "-");
|
||||
|
||||
result = result.Trim('-');
|
||||
if (result.Length > maxLength) result = result.Substring(0, maxLength);
|
||||
result = result.Trim('-');
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
namespace Core
|
||||
{
|
||||
public interface IK8sTimeSet
|
||||
{
|
||||
/// <summary>
|
||||
/// After a failed K8s operation, wait this long before trying again.
|
||||
/// </summary>
|
||||
TimeSpan K8sOperationRetryDelay();
|
||||
|
||||
/// <summary>
|
||||
/// Maximum total time to attempt to perform a successful k8s operation.
|
||||
/// If k8s operations fail during this timespan, retries will be made.
|
||||
/// </summary>
|
||||
TimeSpan K8sOperationTimeout();
|
||||
}
|
||||
|
||||
public class DefaultK8sTimeSet : IK8sTimeSet
|
||||
{
|
||||
public TimeSpan K8sOperationRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(10);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(30);
|
||||
}
|
||||
}
|
||||
|
||||
public class LongK8sTimeSet : IK8sTimeSet
|
||||
{
|
||||
public TimeSpan K8sOperationRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromHours(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<RootNamespace>KubernetesWorkflow</RootNamespace>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
|
||||
@@ -25,11 +25,11 @@ namespace KubernetesWorkflow
|
||||
|
||||
public class WriteToFileLogHandler : LogHandler, ILogHandler
|
||||
{
|
||||
public WriteToFileLogHandler(ILog sourceLog, string description)
|
||||
public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
|
||||
{
|
||||
LogFile = sourceLog.CreateSubfile();
|
||||
LogFile = sourceLog.CreateSubfile(addFileName);
|
||||
|
||||
var msg = $"{description} -->> {LogFile.FullFilename}";
|
||||
var msg = $"{description} -->> {LogFile.Filename}";
|
||||
sourceLog.Log(msg);
|
||||
|
||||
LogFile.Write(msg);
|
||||
|
||||
@@ -13,7 +13,7 @@ namespace KubernetesWorkflow
|
||||
FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
|
||||
PodInfo GetPodInfo(RunningContainer container);
|
||||
PodInfo GetPodInfo(RunningPod pod);
|
||||
CrashWatcher CreateCrashWatcher(RunningContainer container);
|
||||
ContainerCrashWatcher CreateCrashWatcher(RunningContainer container);
|
||||
void Stop(RunningPod pod, bool waitTillStopped);
|
||||
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
|
||||
IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
|
||||
@@ -61,7 +61,8 @@ namespace KubernetesWorkflow
|
||||
var startResult = controller.BringOnline(recipes, location);
|
||||
var containers = CreateContainers(startResult, recipes, startupConfig);
|
||||
|
||||
var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
|
||||
var info = GetPodInfo(startResult.Deployment);
|
||||
var rc = new RunningPod(Guid.NewGuid().ToString(), info, startupConfig, startResult, containers);
|
||||
cluster.Configuration.Hooks.OnContainersStarted(rc);
|
||||
|
||||
if (startResult.ExternalService != null)
|
||||
@@ -83,9 +84,14 @@ namespace KubernetesWorkflow
|
||||
});
|
||||
}
|
||||
|
||||
public PodInfo GetPodInfo(RunningDeployment deployment)
|
||||
{
|
||||
return K8s(c => c.GetPodInfo(deployment));
|
||||
}
|
||||
|
||||
public PodInfo GetPodInfo(RunningContainer container)
|
||||
{
|
||||
return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
|
||||
return GetPodInfo(container.RunningPod.StartResult.Deployment);
|
||||
}
|
||||
|
||||
public PodInfo GetPodInfo(RunningPod pod)
|
||||
@@ -93,7 +99,7 @@ namespace KubernetesWorkflow
|
||||
return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
|
||||
}
|
||||
|
||||
public CrashWatcher CreateCrashWatcher(RunningContainer container)
|
||||
public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
|
||||
{
|
||||
return K8s(c => c.CreateCrashWatcher(container));
|
||||
}
|
||||
@@ -127,14 +133,14 @@ namespace KubernetesWorkflow
|
||||
{
|
||||
var msg = $"Downloading container log for '{container.Name}'";
|
||||
log.Log(msg);
|
||||
var logHandler = new WriteToFileLogHandler(log, msg);
|
||||
var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
|
||||
|
||||
K8s(controller =>
|
||||
{
|
||||
controller.DownloadPodLog(container, logHandler, tailLines, previous);
|
||||
});
|
||||
|
||||
return new DownloadedLog(logHandler, container.Name);
|
||||
return new DownloadedLog(logHandler.LogFile, container.Name);
|
||||
}
|
||||
|
||||
public string ExecuteCommand(RunningContainer container, string command, params string[] args)
|
||||
@@ -209,6 +215,7 @@ namespace KubernetesWorkflow
|
||||
var port = startResult.GetExternalServicePorts(recipe, tag);
|
||||
|
||||
return new Address(
|
||||
logName: $"{recipe.Name}:{tag}",
|
||||
startResult.Cluster.HostAddress,
|
||||
port.Number);
|
||||
}
|
||||
@@ -220,6 +227,7 @@ namespace KubernetesWorkflow
|
||||
var port = startResult.GetInternalServicePorts(recipe, tag);
|
||||
|
||||
return new Address(
|
||||
logName: $"{serviceName}:{tag}",
|
||||
$"http://{serviceName}.{namespaceName}.svc.cluster.local",
|
||||
port.Number);
|
||||
}
|
||||
|
||||
@@ -24,13 +24,12 @@ namespace KubernetesWorkflow.Types
|
||||
[JsonIgnore]
|
||||
public RunningPod RunningPod { get; internal set; } = null!;
|
||||
|
||||
public Address GetAddress(ILog log, string portTag)
|
||||
public Address GetAddress(string portTag)
|
||||
{
|
||||
var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
|
||||
if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
|
||||
|
||||
var select = SelectAddress(addresses);
|
||||
log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
|
||||
return select.Address;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,9 +4,10 @@ namespace KubernetesWorkflow.Types
|
||||
{
|
||||
public class RunningPod
|
||||
{
|
||||
public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
|
||||
public RunningPod(string id, PodInfo podInfo, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
|
||||
{
|
||||
Id = id;
|
||||
PodInfo = podInfo;
|
||||
StartupConfig = startupConfig;
|
||||
StartResult = startResult;
|
||||
Containers = containers;
|
||||
@@ -15,6 +16,7 @@ namespace KubernetesWorkflow.Types
|
||||
}
|
||||
|
||||
public string Id { get; }
|
||||
public PodInfo PodInfo { get; }
|
||||
public StartupConfig StartupConfig { get; }
|
||||
public StartResult StartResult { get; }
|
||||
public RunningContainer[] Containers { get; }
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace Logging
|
||||
void Debug(string message = "", int skipFrames = 0);
|
||||
void Error(string message);
|
||||
void AddStringReplace(string from, string to);
|
||||
LogFile CreateSubfile(string ext = "log");
|
||||
LogFile CreateSubfile(string addName, string ext = "log");
|
||||
}
|
||||
|
||||
public abstract class BaseLog : ILog
|
||||
@@ -34,7 +34,7 @@ namespace Logging
|
||||
{
|
||||
get
|
||||
{
|
||||
if (logFile == null) logFile = new LogFile(GetFullName(), "log");
|
||||
if (logFile == null) logFile = new LogFile(GetFullName() + ".log");
|
||||
return logFile;
|
||||
}
|
||||
}
|
||||
@@ -69,12 +69,16 @@ namespace Logging
|
||||
|
||||
public virtual void Delete()
|
||||
{
|
||||
File.Delete(LogFile.FullFilename);
|
||||
File.Delete(LogFile.Filename);
|
||||
}
|
||||
|
||||
public LogFile CreateSubfile(string ext = "log")
|
||||
public LogFile CreateSubfile(string addName, string ext = "log")
|
||||
{
|
||||
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
|
||||
addName = addName
|
||||
.Replace("<", "")
|
||||
.Replace(">", "");
|
||||
|
||||
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}.{ext}");
|
||||
}
|
||||
|
||||
protected string ApplyReplacements(string str)
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
namespace Logging
|
||||
{
|
||||
public interface IDownloadedLog
|
||||
{
|
||||
string SourceName { get; }
|
||||
|
||||
void IterateLines(Action<string> action);
|
||||
void IterateLines(Action<string> action, params string[] thatContain);
|
||||
string[] GetLinesContaining(string expectedString);
|
||||
string[] FindLinesThatContain(params string[] tags);
|
||||
string GetFilepath();
|
||||
void DeleteFile();
|
||||
}
|
||||
|
||||
public class DownloadedLog : IDownloadedLog
|
||||
{
|
||||
private readonly LogFile logFile;
|
||||
|
||||
public DownloadedLog(string filepath, string sourceName)
|
||||
{
|
||||
logFile = new LogFile(filepath);
|
||||
SourceName = sourceName;
|
||||
}
|
||||
|
||||
public DownloadedLog(LogFile logFile, string sourceName)
|
||||
{
|
||||
this.logFile = logFile;
|
||||
SourceName = sourceName;
|
||||
}
|
||||
|
||||
public string SourceName { get; }
|
||||
|
||||
public void IterateLines(Action<string> action)
|
||||
{
|
||||
using var file = File.OpenRead(logFile.Filename);
|
||||
using var streamReader = new StreamReader(file);
|
||||
|
||||
var line = streamReader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
action(line);
|
||||
line = streamReader.ReadLine();
|
||||
}
|
||||
}
|
||||
|
||||
public void IterateLines(Action<string> action, params string[] thatContain)
|
||||
{
|
||||
IterateLines(line =>
|
||||
{
|
||||
if (thatContain.All(line.Contains))
|
||||
{
|
||||
action(line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public string[] GetLinesContaining(string expectedString)
|
||||
{
|
||||
return FindLinesThatContain([expectedString]);
|
||||
}
|
||||
|
||||
public string[] FindLinesThatContain(params string[] tags)
|
||||
{
|
||||
var result = new List<string>();
|
||||
IterateLines(result.Add, tags);
|
||||
return result.ToArray();
|
||||
}
|
||||
|
||||
public string GetFilepath()
|
||||
{
|
||||
return logFile.Filename;
|
||||
}
|
||||
|
||||
public void DeleteFile()
|
||||
{
|
||||
File.Delete(logFile.Filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,21 +1,19 @@
|
||||
namespace Logging
|
||||
using Utils;
|
||||
|
||||
namespace Logging
|
||||
{
|
||||
public class LogFile
|
||||
{
|
||||
private readonly string extension;
|
||||
private readonly object fileLock = new object();
|
||||
private string filename;
|
||||
|
||||
public LogFile(string filename, string extension)
|
||||
public LogFile(string filename)
|
||||
{
|
||||
this.filename = filename;
|
||||
this.extension = extension;
|
||||
FullFilename = filename + "." + extension;
|
||||
Filename = filename;
|
||||
|
||||
EnsurePathExists(filename);
|
||||
}
|
||||
|
||||
public string FullFilename { get; private set; }
|
||||
public string Filename { get; private set; }
|
||||
|
||||
public void Write(string message)
|
||||
{
|
||||
@@ -28,7 +26,7 @@
|
||||
{
|
||||
lock (fileLock)
|
||||
{
|
||||
File.AppendAllLines(FullFilename, new[] { message });
|
||||
File.AppendAllLines(Filename, new[] { message });
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -37,19 +35,24 @@
|
||||
}
|
||||
}
|
||||
|
||||
public void ConcatToFilename(string toAdd)
|
||||
public void WriteRawMany(IEnumerable<string> lines)
|
||||
{
|
||||
var oldFullName = FullFilename;
|
||||
|
||||
filename += toAdd;
|
||||
FullFilename = filename + "." + extension;
|
||||
|
||||
File.Move(oldFullName, FullFilename);
|
||||
try
|
||||
{
|
||||
lock (fileLock)
|
||||
{
|
||||
File.AppendAllLines(Filename, lines);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine("Writing to log has failed: " + ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetTimestamp()
|
||||
{
|
||||
return $"[{DateTime.UtcNow.ToString("o")}]";
|
||||
return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
|
||||
}
|
||||
|
||||
private void EnsurePathExists(string filename)
|
||||
|
||||
@@ -18,9 +18,9 @@
|
||||
public string Prefix { get; set; } = string.Empty;
|
||||
|
||||
|
||||
public LogFile CreateSubfile(string ext = "log")
|
||||
public LogFile CreateSubfile(string addName, string ext = "log")
|
||||
{
|
||||
return backingLog.CreateSubfile(ext);
|
||||
return backingLog.CreateSubfile(addName, ext);
|
||||
}
|
||||
|
||||
public void Debug(string message = "", int skipFrames = 0)
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
OnAll(l => l.AddStringReplace(from, to));
|
||||
}
|
||||
|
||||
public LogFile CreateSubfile(string ext = "log")
|
||||
public LogFile CreateSubfile(string addName, string ext = "log")
|
||||
{
|
||||
return targetLogs.First().CreateSubfile(ext);
|
||||
return targetLogs.First().CreateSubfile(addName, ext);
|
||||
}
|
||||
|
||||
public void Debug(string message = "", int skipFrames = 0)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<RootNamespace>Logging</RootNamespace>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
using Logging;
|
||||
using BlockchainUtils;
|
||||
using Logging;
|
||||
using Nethereum.ABI.FunctionEncoding.Attributes;
|
||||
using Nethereum.Contracts;
|
||||
using Nethereum.RPC.Eth.DTOs;
|
||||
using Nethereum.Web3;
|
||||
using NethereumWorkflow.BlockUtils;
|
||||
using Utils;
|
||||
|
||||
namespace NethereumWorkflow
|
||||
{
|
||||
public class NethereumInteraction
|
||||
{
|
||||
// BlockCache is a static instance: It stays alive for the duration of the application runtime.
|
||||
private readonly static BlockCache blockCache = new BlockCache();
|
||||
private readonly BlockCache blockCache;
|
||||
|
||||
private readonly ILog log;
|
||||
private readonly Web3 web3;
|
||||
|
||||
internal NethereumInteraction(ILog log, Web3 web3)
|
||||
internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
|
||||
{
|
||||
this.log = log;
|
||||
this.web3 = web3;
|
||||
this.blockCache = blockCache;
|
||||
}
|
||||
|
||||
public string SendEth(string toAddress, decimal ethAmount)
|
||||
@@ -50,6 +50,21 @@ namespace NethereumWorkflow
|
||||
return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
|
||||
}
|
||||
|
||||
public TResult Call<TFunction, TResult>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
|
||||
{
|
||||
log.Debug(typeof(TFunction).ToString());
|
||||
var handler = web3.Eth.GetContractQueryHandler<TFunction>();
|
||||
return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function, new BlockParameter(blockNumber)));
|
||||
}
|
||||
|
||||
public void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
|
||||
{
|
||||
log.Debug(typeof(TFunction).ToString());
|
||||
var handler = web3.Eth.GetContractQueryHandler<TFunction>();
|
||||
var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber)));
|
||||
var aaaa = 0;
|
||||
}
|
||||
|
||||
public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
|
||||
{
|
||||
log.Debug();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using Logging;
|
||||
using BlockchainUtils;
|
||||
using Logging;
|
||||
using Nethereum.Web3;
|
||||
|
||||
namespace NethereumWorkflow
|
||||
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
|
||||
public class NethereumInteractionCreator
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly BlockCache blockCache;
|
||||
private readonly string ip;
|
||||
private readonly int port;
|
||||
private readonly string privateKey;
|
||||
|
||||
public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
|
||||
public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
|
||||
{
|
||||
this.log = log;
|
||||
this.blockCache = blockCache;
|
||||
this.ip = ip;
|
||||
this.port = port;
|
||||
this.privateKey = privateKey;
|
||||
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
|
||||
public NethereumInteraction CreateWorkflow()
|
||||
{
|
||||
log.Debug("Starting interaction to " + ip + ":" + port);
|
||||
return new NethereumInteraction(log, CreateWeb3());
|
||||
return new NethereumInteraction(log, CreateWeb3(), blockCache);
|
||||
}
|
||||
|
||||
private Web3 CreateWeb3()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<RootNamespace>NethereumWorkflow</RootNamespace>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
@@ -12,6 +12,7 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
|
||||
<ProjectReference Include="..\Logging\Logging.csproj" />
|
||||
<ProjectReference Include="..\Utils\Utils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,16 +1,11 @@
|
||||
using Logging;
|
||||
using BlockchainUtils;
|
||||
using Logging;
|
||||
using Nethereum.RPC.Eth.DTOs;
|
||||
using Nethereum.Web3;
|
||||
using Utils;
|
||||
|
||||
namespace NethereumWorkflow
|
||||
{
|
||||
public interface IWeb3Blocks
|
||||
{
|
||||
ulong GetCurrentBlockNumber();
|
||||
DateTime? GetTimestampForBlock(ulong blockNumber);
|
||||
}
|
||||
|
||||
public class Web3Wrapper : IWeb3Blocks
|
||||
{
|
||||
private readonly Web3 web3;
|
||||
|
||||
@@ -7,6 +7,7 @@ namespace OverwatchTranscript
|
||||
public interface IFinalizedBucket
|
||||
{
|
||||
bool IsEmpty { get; }
|
||||
void Update();
|
||||
DateTime? SeeTopUtc();
|
||||
BucketTop? TakeTop();
|
||||
}
|
||||
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
|
||||
private readonly string bucketFile;
|
||||
private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
|
||||
private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
|
||||
private bool stopping;
|
||||
private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
|
||||
private bool sourceIsEmpty;
|
||||
|
||||
public EventBucketReader(ILog log, string bucketFile)
|
||||
{
|
||||
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
|
||||
|
||||
public bool IsEmpty { get; private set; }
|
||||
|
||||
public void Update()
|
||||
{
|
||||
if (IsEmpty) return;
|
||||
while (topQueue.Count == 0)
|
||||
{
|
||||
UpdateIsEmpty();
|
||||
if (IsEmpty) return;
|
||||
|
||||
itemDequeued.Set();
|
||||
itemEnqueued.WaitOne(200);
|
||||
}
|
||||
}
|
||||
|
||||
public DateTime? SeeTopUtc()
|
||||
{
|
||||
if (IsEmpty) return null;
|
||||
while (true)
|
||||
if (topQueue.TryPeek(out BucketTop? top))
|
||||
{
|
||||
UpdateIsEmpty();
|
||||
if (IsEmpty) return null;
|
||||
if (topQueue.TryPeek(out BucketTop? top))
|
||||
{
|
||||
return top.Utc;
|
||||
}
|
||||
return top.Utc;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public BucketTop? TakeTop()
|
||||
{
|
||||
if (IsEmpty) return null;
|
||||
|
||||
while (true)
|
||||
if (topQueue.TryDequeue(out BucketTop? top))
|
||||
{
|
||||
UpdateIsEmpty();
|
||||
if (IsEmpty) return null;
|
||||
if (topQueue.TryDequeue(out BucketTop? top))
|
||||
{
|
||||
itemDequeued.Set();
|
||||
return top;
|
||||
}
|
||||
itemDequeued.Set();
|
||||
return top;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void ReadBucket()
|
||||
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
|
||||
if (top != null)
|
||||
{
|
||||
topQueue.Enqueue(top);
|
||||
itemEnqueued.Set();
|
||||
}
|
||||
else
|
||||
{
|
||||
stopping = true;
|
||||
sourceIsEmpty = true;
|
||||
UpdateIsEmpty();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
itemDequeued.Reset();
|
||||
itemDequeued.WaitOne();
|
||||
itemDequeued.WaitOne(5000);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateIsEmpty()
|
||||
{
|
||||
var empty = stopping && topQueue.IsEmpty;
|
||||
if (!IsEmpty && empty)
|
||||
var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
|
||||
if (!IsEmpty && allEmpty)
|
||||
{
|
||||
File.Delete(bucketFile);
|
||||
IsEmpty = true;
|
||||
|
||||
@@ -24,6 +24,8 @@ namespace OverwatchTranscript
|
||||
log.Debug($"Building references for {buckets.Count} buckets.");
|
||||
while (buckets.Any())
|
||||
{
|
||||
foreach (var b in buckets) b.Update();
|
||||
|
||||
buckets.RemoveAll(b => b.IsEmpty);
|
||||
if (!buckets.Any()) break;
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
{
|
||||
public class Address
|
||||
{
|
||||
public Address(string host, int port)
|
||||
public Address(string logName, string host, int port)
|
||||
{
|
||||
LogName = logName;
|
||||
Host = host;
|
||||
Port = port;
|
||||
}
|
||||
|
||||
public string LogName { get; }
|
||||
public string Host { get; }
|
||||
public int Port { get; }
|
||||
|
||||
@@ -20,5 +22,21 @@
|
||||
{
|
||||
return !string.IsNullOrEmpty(Host) && Port > 0;
|
||||
}
|
||||
|
||||
public static Address Empty()
|
||||
{
|
||||
return new Address(string.Empty, string.Empty, 0);
|
||||
}
|
||||
}
|
||||
|
||||
public interface IHasMetricsScrapeTarget
|
||||
{
|
||||
Address GetMetricsScrapeTarget();
|
||||
}
|
||||
|
||||
public interface IHasManyMetricScrapeTargets
|
||||
{
|
||||
Address[] GetMetricsScrapeTargets();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Contracts;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Utils
|
||||
{
|
||||
public static class Base58
|
||||
{
|
||||
private const string Digits = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
|
||||
|
||||
public static string Encode(byte[] data)
|
||||
{
|
||||
// Decode byte[] to BigInteger
|
||||
BigInteger intData = 0;
|
||||
for (int i = 0; i < data.Length; i++)
|
||||
{
|
||||
intData = intData * 256 + data[i];
|
||||
}
|
||||
|
||||
// Encode BigInteger to Base58 string
|
||||
string result = "";
|
||||
while (intData > 0)
|
||||
{
|
||||
int remainder = (int)(intData % 58);
|
||||
intData /= 58;
|
||||
result = Digits[remainder] + result;
|
||||
}
|
||||
|
||||
// Append `1` for each leading 0 byte
|
||||
for (int i = 0; i < data.Length && data[i] == 0; i++)
|
||||
{
|
||||
result = '1' + result;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static byte[] Decode(string s)
|
||||
{
|
||||
BigInteger intData = 0;
|
||||
for (int i = 0; i < s.Length; i++)
|
||||
{
|
||||
int digit = Digits.IndexOf(s[i]); //Slow
|
||||
if (digit < 0)
|
||||
throw new FormatException(string.Format("Invalid Base58 character `{0}` at position {1}", s[i], i));
|
||||
intData = intData * 58 + digit;
|
||||
}
|
||||
|
||||
// Encode BigInteger to byte[]
|
||||
// Leading zero bytes get encoded as leading `1` characters
|
||||
int leadingZeroCount = s.TakeWhile(c => c == '1').Count();
|
||||
var leadingZeros = Enumerable.Repeat((byte)0, leadingZeroCount);
|
||||
var bytesWithoutLeadingZeros =
|
||||
intData.ToByteArray()
|
||||
.Reverse()// to big endian
|
||||
.SkipWhile(b => b == 0);//strip sign byte
|
||||
var result = leadingZeros.Concat(bytesWithoutLeadingZeros).ToArray();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,12 +15,13 @@
|
||||
To = from;
|
||||
}
|
||||
TimeRange = timeRange;
|
||||
NumberOfBlocks = (To - From) + 1;
|
||||
}
|
||||
|
||||
public ulong From { get; }
|
||||
public ulong To { get; }
|
||||
public TimeRange TimeRange { get; }
|
||||
public ulong NumberOfBlocks => To - From;
|
||||
public ulong NumberOfBlocks { get; }
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace Utils
|
||||
{
|
||||
//public interface ICrashWatcher
|
||||
//{
|
||||
// void Start();
|
||||
// void Stop();
|
||||
// bool HasCrashed();
|
||||
//}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace Utils
|
||||
{
|
||||
[Serializable]
|
||||
public class EthAccount
|
||||
{
|
||||
public EthAccount(EthAddress ethAddress, string privateKey)
|
||||
{
|
||||
EthAddress = ethAddress;
|
||||
PrivateKey = privateKey;
|
||||
}
|
||||
|
||||
public EthAddress EthAddress { get; }
|
||||
public string PrivateKey { get; }
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return EthAddress.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
namespace GethPlugin
|
||||
namespace Utils
|
||||
{
|
||||
public interface IHasEthAddress
|
||||
{
|
||||
+8
-2
@@ -1,4 +1,4 @@
|
||||
namespace GethPlugin
|
||||
namespace Utils
|
||||
{
|
||||
public class Ether : IComparable<Ether>
|
||||
{
|
||||
@@ -28,7 +28,13 @@
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{Eth} Eth";
|
||||
var weiOnly = Wei % TokensIntExtensions.WeiPerEth;
|
||||
|
||||
var tokens = new List<string>();
|
||||
if (Eth > 0) tokens.Add($"{Eth} Eth");
|
||||
if (weiOnly > 0) tokens.Add($"{weiOnly} Wei");
|
||||
|
||||
return string.Join(" + ", tokens);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
namespace Utils
|
||||
{
|
||||
public static class PluginPathUtils
|
||||
{
|
||||
private const string ProjectPluginsFolderName = "ProjectPlugins";
|
||||
private static string projectPluginsDir = string.Empty;
|
||||
|
||||
public static string ProjectPluginsDir
|
||||
{
|
||||
get
|
||||
{
|
||||
if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
|
||||
return projectPluginsDir;
|
||||
}
|
||||
}
|
||||
|
||||
private static string FindProjectPluginsDir()
|
||||
{
|
||||
var current = Directory.GetCurrentDirectory();
|
||||
while (true)
|
||||
{
|
||||
var localFolders = Directory.GetDirectories(current);
|
||||
var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
|
||||
if (projectPluginsFolders.Length == 1)
|
||||
{
|
||||
return projectPluginsFolders.Single();
|
||||
}
|
||||
|
||||
var parent = Directory.GetParent(current);
|
||||
if (parent == null)
|
||||
{
|
||||
var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
|
||||
Console.WriteLine(msg);
|
||||
throw new Exception(msg);
|
||||
}
|
||||
|
||||
current = parent.FullName;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,24 +3,41 @@
|
||||
public static class RandomUtils
|
||||
{
|
||||
private static readonly Random random = new Random();
|
||||
private static readonly object @lock = new object();
|
||||
|
||||
public static T GetOneRandom<T>(this T[] items)
|
||||
{
|
||||
lock (@lock)
|
||||
{
|
||||
var i = random.Next(0, items.Length);
|
||||
var result = items[i];
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public static T PickOneRandom<T>(this List<T> remainingItems)
|
||||
{
|
||||
var i = random.Next(0, remainingItems.Count);
|
||||
var result = remainingItems[i];
|
||||
remainingItems.RemoveAt(i);
|
||||
return result;
|
||||
lock (@lock)
|
||||
{
|
||||
var i = random.Next(0, remainingItems.Count);
|
||||
var result = remainingItems[i];
|
||||
remainingItems.RemoveAt(i);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public static T[] Shuffled<T>(T[] items)
|
||||
{
|
||||
var result = new List<T>();
|
||||
var source = items.ToList();
|
||||
while (source.Any())
|
||||
lock (@lock)
|
||||
{
|
||||
result.Add(RandomUtils.PickOneRandom(source));
|
||||
var result = new List<T>();
|
||||
var source = items.ToList();
|
||||
while (source.Any())
|
||||
{
|
||||
result.Add(RandomUtils.PickOneRandom(source));
|
||||
}
|
||||
return result.ToArray();
|
||||
}
|
||||
return result.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,6 +71,10 @@
|
||||
task();
|
||||
return;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var failure = CaptureFailure(ex);
|
||||
@@ -94,7 +98,7 @@
|
||||
|
||||
private void Fail()
|
||||
{
|
||||
throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
|
||||
throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
|
||||
new AggregateException(failures.Select(f => f.Exception)));
|
||||
}
|
||||
|
||||
|
||||
+22
-2
@@ -1,6 +1,6 @@
|
||||
using System.Numerics;
|
||||
|
||||
namespace CodexContractsPlugin
|
||||
namespace Utils
|
||||
{
|
||||
public class TestToken : IComparable<TestToken>
|
||||
{
|
||||
@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
|
||||
return new TestToken(a.TstWei + b.TstWei);
|
||||
}
|
||||
|
||||
public static TestToken operator -(TestToken a, TestToken b)
|
||||
{
|
||||
return new TestToken(a.TstWei - b.TstWei);
|
||||
}
|
||||
|
||||
public static TestToken operator *(TestToken a, int b)
|
||||
{
|
||||
return new TestToken(a.TstWei * b);
|
||||
}
|
||||
|
||||
public static bool operator <(TestToken a, TestToken b)
|
||||
{
|
||||
return a.TstWei < b.TstWei;
|
||||
@@ -71,7 +81,7 @@ namespace CodexContractsPlugin
|
||||
{
|
||||
public static TestToken TstWei(this int i)
|
||||
{
|
||||
return TstWei(Convert.ToDecimal(i));
|
||||
return new TestToken(new BigInteger(i));
|
||||
}
|
||||
|
||||
public static TestToken TstWei(this decimal i)
|
||||
@@ -84,6 +94,11 @@ namespace CodexContractsPlugin
|
||||
return new TestToken(i);
|
||||
}
|
||||
|
||||
public static TestToken TstWei(this string s)
|
||||
{
|
||||
return new TestToken(BigInteger.Parse(s));
|
||||
}
|
||||
|
||||
public static TestToken Tst(this int i)
|
||||
{
|
||||
return Tst(Convert.ToDecimal(i));
|
||||
@@ -98,5 +113,10 @@ namespace CodexContractsPlugin
|
||||
{
|
||||
return new TestToken(i * TestToken.WeiFactor);
|
||||
}
|
||||
|
||||
public static TestToken Tst(this string s)
|
||||
{
|
||||
return new TestToken(BigInteger.Parse(s) * TestToken.WeiFactor);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,11 @@
|
||||
result += $"{d.Seconds} secs";
|
||||
return result;
|
||||
}
|
||||
|
||||
public static string FormatTimestamp(DateTime d)
|
||||
{
|
||||
return d.ToString("o");
|
||||
}
|
||||
|
||||
public static TimeSpan ParseTimespan(string span)
|
||||
{
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<RootNamespace>Utils</RootNamespace>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
using Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Serialization = Newtonsoft.Json.Serialization;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Utils;
|
||||
|
||||
namespace Core
|
||||
namespace WebUtils
|
||||
{
|
||||
public interface IEndpoint
|
||||
{
|
||||
@@ -119,7 +118,7 @@ namespace Core
|
||||
var errors = new List<string>();
|
||||
var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
|
||||
{
|
||||
Error = delegate (object? sender, Serialization.ErrorEventArgs args)
|
||||
Error = delegate (object? sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
|
||||
{
|
||||
if (args.CurrentObject == args.ErrorContext.OriginalObject)
|
||||
{
|
||||
@@ -1,7 +1,7 @@
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace Core
|
||||
namespace WebUtils
|
||||
{
|
||||
public interface IHttp
|
||||
{
|
||||
@@ -16,16 +16,16 @@ namespace Core
|
||||
private static object lockLock = new object();
|
||||
private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
|
||||
private readonly ILog log;
|
||||
private readonly ITimeSet timeSet;
|
||||
private readonly IWebCallTimeSet timeSet;
|
||||
private readonly Action<HttpClient> onClientCreated;
|
||||
private readonly string id;
|
||||
|
||||
internal Http(string id, ILog log, ITimeSet timeSet)
|
||||
internal Http(string id, ILog log, IWebCallTimeSet timeSet)
|
||||
: this(id, log, timeSet, DoNothing)
|
||||
{
|
||||
}
|
||||
|
||||
internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
|
||||
internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action<HttpClient> onClientCreated)
|
||||
{
|
||||
this.id = id;
|
||||
this.log = log;
|
||||
@@ -0,0 +1,43 @@
|
||||
using Logging;
|
||||
|
||||
namespace WebUtils
|
||||
{
|
||||
public interface IHttpFactory
|
||||
{
|
||||
IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
|
||||
IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet);
|
||||
IHttp CreateHttp(string id);
|
||||
}
|
||||
|
||||
public class HttpFactory : IHttpFactory
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly IWebCallTimeSet defaultTimeSet;
|
||||
|
||||
public HttpFactory(ILog log)
|
||||
: this (log, new DefaultWebCallTimeSet())
|
||||
{
|
||||
}
|
||||
|
||||
public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet)
|
||||
{
|
||||
this.log = log;
|
||||
this.defaultTimeSet = defaultTimeSet;
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
|
||||
{
|
||||
return CreateHttp(id, onClientCreated, defaultTimeSet);
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet ts)
|
||||
{
|
||||
return new Http(id, log, ts, onClientCreated);
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(string id)
|
||||
{
|
||||
return new Http(id, log, defaultTimeSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
namespace Core
|
||||
namespace WebUtils
|
||||
{
|
||||
public interface ITimeSet
|
||||
public interface IWebCallTimeSet
|
||||
{
|
||||
/// <summary>
|
||||
/// Timeout for a single HTTP call.
|
||||
@@ -17,20 +17,9 @@
|
||||
/// After a failed HTTP call, wait this long before trying again.
|
||||
/// </summary>
|
||||
TimeSpan HttpCallRetryDelay();
|
||||
|
||||
/// <summary>
|
||||
/// After a failed K8s operation, wait this long before trying again.
|
||||
/// </summary>
|
||||
TimeSpan K8sOperationRetryDelay();
|
||||
|
||||
/// <summary>
|
||||
/// Maximum total time to attempt to perform a successful k8s operation.
|
||||
/// If k8s operations fail during this timespan, retries will be made.
|
||||
/// </summary>
|
||||
TimeSpan K8sOperationTimeout();
|
||||
}
|
||||
|
||||
public class DefaultTimeSet : ITimeSet
|
||||
public class DefaultWebCallTimeSet : IWebCallTimeSet
|
||||
{
|
||||
public TimeSpan HttpCallTimeout()
|
||||
{
|
||||
@@ -46,19 +35,9 @@
|
||||
{
|
||||
return TimeSpan.FromSeconds(1);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(10);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(30);
|
||||
}
|
||||
}
|
||||
|
||||
public class LongTimeSet : ITimeSet
|
||||
public class LongWebCallTimeSet : IWebCallTimeSet
|
||||
{
|
||||
public TimeSpan HttpCallTimeout()
|
||||
{
|
||||
@@ -74,15 +53,5 @@
|
||||
{
|
||||
return TimeSpan.FromSeconds(20);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromHours(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Logging\Logging.csproj" />
|
||||
<ProjectReference Include="..\Utils\Utils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,314 @@
|
||||
using CodexOpenApi;
|
||||
using Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Utils;
|
||||
using WebUtils;
|
||||
|
||||
namespace CodexClient
|
||||
{
|
||||
public class CodexAccess
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly IHttpFactory httpFactory;
|
||||
private readonly IProcessControl processControl;
|
||||
private ICodexInstance instance;
|
||||
private readonly Mapper mapper = new Mapper();
|
||||
|
||||
public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
|
||||
{
|
||||
this.log = log;
|
||||
this.httpFactory = httpFactory;
|
||||
this.processControl = processControl;
|
||||
this.instance = instance;
|
||||
}
|
||||
|
||||
public void Stop(bool waitTillStopped)
|
||||
{
|
||||
processControl.Stop(waitTillStopped);
|
||||
// Prevents accidental use after stop:
|
||||
instance = null!;
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(string additionalName = "")
|
||||
{
|
||||
var file = log.CreateSubfile(GetName() + additionalName);
|
||||
Log($"Downloading logs to '{file.Filename}'");
|
||||
return processControl.DownloadLog(file);
|
||||
}
|
||||
|
||||
public string GetImageName()
|
||||
{
|
||||
return instance.ImageName;
|
||||
}
|
||||
|
||||
public DateTime GetStartUtc()
|
||||
{
|
||||
return instance.StartUtc;
|
||||
}
|
||||
|
||||
public DebugInfo GetDebugInfo()
|
||||
{
|
||||
return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
|
||||
}
|
||||
|
||||
public void SetLogLevel(string logLevel)
|
||||
{
|
||||
try
|
||||
{
|
||||
OnCodex(async api =>
|
||||
{
|
||||
await api.SetDebugLogLevelAsync(logLevel);
|
||||
return string.Empty;
|
||||
});
|
||||
}
|
||||
catch (Exception exc)
|
||||
{
|
||||
log.Error("Failed to set log level: " + exc);
|
||||
}
|
||||
}
|
||||
|
||||
public string GetSpr()
|
||||
{
|
||||
return CrashCheck(() =>
|
||||
{
|
||||
var endpoint = GetEndpoint();
|
||||
var json = endpoint.HttpGetString("spr");
|
||||
var response = JsonConvert.DeserializeObject<SprResponse>(json);
|
||||
return response!.Spr;
|
||||
});
|
||||
}
|
||||
|
||||
private class SprResponse
|
||||
{
|
||||
public string Spr { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public DebugPeer GetDebugPeer(string peerId)
|
||||
{
|
||||
// Cannot use openAPI: debug/peer endpoint is not specified there.
|
||||
return CrashCheck(() =>
|
||||
{
|
||||
var endpoint = GetEndpoint();
|
||||
var str = endpoint.HttpGetString($"debug/peer/{peerId}");
|
||||
|
||||
if (str.ToLowerInvariant() == "unable to find peer!")
|
||||
{
|
||||
return new DebugPeer
|
||||
{
|
||||
IsPeerFound = false
|
||||
};
|
||||
}
|
||||
|
||||
var result = endpoint.Deserialize<DebugPeer>(str);
|
||||
result.IsPeerFound = true;
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
|
||||
{
|
||||
OnCodex(api =>
|
||||
{
|
||||
Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
|
||||
return Task.FromResult(string.Empty);
|
||||
});
|
||||
}
|
||||
|
||||
public string UploadFile(UploadInput uploadInput)
|
||||
{
|
||||
return OnCodex(api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream));
|
||||
}
|
||||
|
||||
public Stream DownloadFile(string contentId)
|
||||
{
|
||||
var fileResponse = OnCodex(api => api.DownloadNetworkStreamAsync(contentId));
|
||||
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
|
||||
return fileResponse.Stream;
|
||||
}
|
||||
|
||||
public LocalDataset DownloadStreamless(ContentId cid)
|
||||
{
|
||||
var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
|
||||
return mapper.Map(response);
|
||||
}
|
||||
|
||||
public LocalDataset DownloadManifestOnly(ContentId cid)
|
||||
{
|
||||
var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
|
||||
return mapper.Map(response);
|
||||
}
|
||||
|
||||
public LocalDatasetList LocalFiles()
|
||||
{
|
||||
return mapper.Map(OnCodex(api => api.ListDataAsync()));
|
||||
}
|
||||
|
||||
public StorageAvailability SalesAvailability(StorageAvailability request)
|
||||
{
|
||||
var body = mapper.Map(request);
|
||||
var read = OnCodex(api => api.OfferStorageAsync(body));
|
||||
return mapper.Map(read);
|
||||
}
|
||||
|
||||
public StorageAvailability[] GetAvailabilities()
|
||||
{
|
||||
var collection = OnCodex(api => api.GetAvailabilitiesAsync());
|
||||
return mapper.Map(collection);
|
||||
}
|
||||
|
||||
public string RequestStorage(StoragePurchaseRequest request)
|
||||
{
|
||||
var body = mapper.Map(request);
|
||||
return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
|
||||
}
|
||||
|
||||
public CodexSpace Space()
|
||||
{
|
||||
var space = OnCodex(api => api.SpaceAsync());
|
||||
return mapper.Map(space);
|
||||
}
|
||||
|
||||
public StoragePurchase? GetPurchaseStatus(string purchaseId)
|
||||
{
|
||||
return CrashCheck(() =>
|
||||
{
|
||||
var endpoint = GetEndpoint();
|
||||
try
|
||||
{
|
||||
return Time.Retry(() =>
|
||||
{
|
||||
var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
|
||||
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
|
||||
return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
|
||||
}, nameof(GetPurchaseStatus));
|
||||
}
|
||||
catch (Exception exc)
|
||||
{
|
||||
log.Error($"Failed to fetch purchase information for id: '{purchaseId}'. Exception: {exc.Message}");
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
// TODO: current getpurchase api does not line up with its openapi spec.
|
||||
// return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
|
||||
}
|
||||
|
||||
public string GetName()
|
||||
{
|
||||
return instance.Name;
|
||||
}
|
||||
|
||||
public Address GetDiscoveryEndpoint()
|
||||
{
|
||||
return instance.DiscoveryEndpoint;
|
||||
}
|
||||
|
||||
public Address GetApiEndpoint()
|
||||
{
|
||||
return instance.ApiEndpoint;
|
||||
}
|
||||
|
||||
public Address GetListenEndpoint()
|
||||
{
|
||||
return instance.ListenEndpoint;
|
||||
}
|
||||
|
||||
public bool HasCrashed()
|
||||
{
|
||||
return processControl.HasCrashed();
|
||||
}
|
||||
|
||||
public Address? GetMetricsEndpoint()
|
||||
{
|
||||
return instance.MetricsEndpoint;
|
||||
}
|
||||
|
||||
public EthAccount? GetEthAccount()
|
||||
{
|
||||
return instance.EthAccount;
|
||||
}
|
||||
|
||||
public void DeleteDataDirFolder()
|
||||
{
|
||||
processControl.DeleteDataDirFolder();
|
||||
}
|
||||
|
||||
private T OnCodex<T>(Func<CodexApiClient, Task<T>> action)
|
||||
{
|
||||
var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action));
|
||||
return result;
|
||||
}
|
||||
|
||||
private T OnCodex<T>(Func<CodexApiClient, Task<T>> action, Retry retry)
|
||||
{
|
||||
var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), retry);
|
||||
return result;
|
||||
}
|
||||
|
||||
private T CallCodex<T>(HttpClient client, Func<CodexApiClient, Task<T>> action)
|
||||
{
|
||||
var address = GetAddress();
|
||||
var api = new CodexApiClient(client);
|
||||
api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
|
||||
return CrashCheck(() => Time.Wait(action(api)));
|
||||
}
|
||||
|
||||
private T CrashCheck<T>(Func<T> action)
|
||||
{
|
||||
try
|
||||
{
|
||||
return action();
|
||||
}
|
||||
finally
|
||||
{
|
||||
CheckContainerCrashed();
|
||||
}
|
||||
}
|
||||
|
||||
private IEndpoint GetEndpoint()
|
||||
{
|
||||
return httpFactory
|
||||
.CreateHttp(GetHttpId(), h => CheckContainerCrashed())
|
||||
.CreateEndpoint(GetAddress(), "/api/codex/v1/", GetName());
|
||||
}
|
||||
|
||||
private Address GetAddress()
|
||||
{
|
||||
return instance.ApiEndpoint;
|
||||
}
|
||||
|
||||
private string GetHttpId()
|
||||
{
|
||||
return GetAddress().ToString();
|
||||
}
|
||||
|
||||
private void CheckContainerCrashed()
|
||||
{
|
||||
if (processControl.HasCrashed()) throw new Exception($"Container {GetName()} has crashed.");
|
||||
}
|
||||
|
||||
private void Throw(Failure failure)
|
||||
{
|
||||
throw failure.Exception;
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
log.Log($"({GetName()}) {msg}");
|
||||
}
|
||||
}
|
||||
|
||||
public class UploadInput
|
||||
{
|
||||
public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
|
||||
{
|
||||
ContentType = contentType;
|
||||
ContentDisposition = contentDisposition;
|
||||
FileStream = fileStream;
|
||||
}
|
||||
|
||||
public string ContentType { get; }
|
||||
public string ContentDisposition { get; }
|
||||
public FileStream FileStream { get; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Remove="openapi.yaml" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApiClient" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Framework\FileUtils\FileUtils.csproj" />
|
||||
<ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
|
||||
<ProjectReference Include="..\..\Framework\WebUtils\WebUtils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,54 @@
|
||||
using Utils;
|
||||
|
||||
namespace CodexClient
|
||||
{
|
||||
public interface ICodexInstance
|
||||
{
|
||||
string Name { get; }
|
||||
string ImageName { get; }
|
||||
DateTime StartUtc { get; }
|
||||
Address DiscoveryEndpoint { get; }
|
||||
Address ApiEndpoint { get; }
|
||||
Address ListenEndpoint { get; }
|
||||
EthAccount? EthAccount { get; }
|
||||
Address? MetricsEndpoint { get; }
|
||||
}
|
||||
|
||||
public class CodexInstance : ICodexInstance
|
||||
{
|
||||
public CodexInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint)
|
||||
{
|
||||
Name = name;
|
||||
ImageName = imageName;
|
||||
StartUtc = startUtc;
|
||||
DiscoveryEndpoint = discoveryEndpoint;
|
||||
ApiEndpoint = apiEndpoint;
|
||||
ListenEndpoint = listenEndpoint;
|
||||
EthAccount = ethAccount;
|
||||
MetricsEndpoint = metricsEndpoint;
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
public string ImageName { get; }
|
||||
public DateTime StartUtc { get; }
|
||||
public Address DiscoveryEndpoint { get; }
|
||||
public Address ApiEndpoint { get; }
|
||||
public Address ListenEndpoint { get; }
|
||||
public EthAccount? EthAccount { get; }
|
||||
public Address? MetricsEndpoint { get; }
|
||||
|
||||
public static ICodexInstance CreateFromApiEndpoint(string name, Address apiEndpoint, EthAccount? ethAccount = null)
|
||||
{
|
||||
return new CodexInstance(
|
||||
name,
|
||||
imageName: "-",
|
||||
startUtc: DateTime.UtcNow,
|
||||
discoveryEndpoint: Address.Empty(),
|
||||
apiEndpoint: apiEndpoint,
|
||||
listenEndpoint: Address.Empty(),
|
||||
ethAccount: ethAccount,
|
||||
metricsEndpoint: null
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public enum CodexLogLevel
|
||||
{
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public class CodexLogLine
|
||||
{
|
||||
@@ -0,0 +1,409 @@
|
||||
using CodexClient.Hooks;
|
||||
using FileUtils;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace CodexClient
|
||||
{
|
||||
public partial interface ICodexNode : IHasEthAddress, IHasMetricsScrapeTarget
|
||||
{
|
||||
string GetName();
|
||||
string GetImageName();
|
||||
string GetPeerId();
|
||||
DebugInfo GetDebugInfo(bool log = false);
|
||||
void SetLogLevel(string logLevel);
|
||||
string GetSpr();
|
||||
DebugPeer GetDebugPeer(string peerId);
|
||||
ContentId UploadFile(TrackedFile file);
|
||||
ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition);
|
||||
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
|
||||
LocalDataset DownloadStreamless(ContentId cid);
|
||||
/// <summary>
|
||||
/// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
|
||||
/// to track the streamless download progress. Replace it once we have a good API for this.
|
||||
/// </summary>
|
||||
LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
|
||||
LocalDataset DownloadManifestOnly(ContentId cid);
|
||||
LocalDatasetList LocalFiles();
|
||||
CodexSpace Space();
|
||||
void ConnectToPeer(ICodexNode node);
|
||||
DebugInfoVersion Version { get; }
|
||||
IMarketplaceAccess Marketplace { get; }
|
||||
ITransferSpeeds TransferSpeeds { get; }
|
||||
EthAccount EthAccount { get; }
|
||||
StoragePurchase? GetPurchaseStatus(string purchaseId);
|
||||
|
||||
Address GetDiscoveryEndpoint();
|
||||
Address GetApiEndpoint();
|
||||
Address GetListenEndpoint();
|
||||
|
||||
/// <summary>
|
||||
/// Warning! The node is not usable after this.
|
||||
/// TODO: Replace with delete-blocks debug call once available in Codex.
|
||||
/// </summary>
|
||||
void DeleteDataDirFolder();
|
||||
void Stop(bool waitTillStopped);
|
||||
IDownloadedLog DownloadLog(string additionalName = "");
|
||||
bool HasCrashed();
|
||||
}
|
||||
|
||||
public class CodexNode : ICodexNode
|
||||
{
|
||||
private const string UploadFailedMessage = "Unable to store block";
|
||||
private readonly ILog log;
|
||||
private readonly ICodexNodeHooks hooks;
|
||||
private readonly TransferSpeeds transferSpeeds;
|
||||
private string peerId = string.Empty;
|
||||
private string nodeId = string.Empty;
|
||||
private readonly CodexAccess codexAccess;
|
||||
private readonly IFileManager fileManager;
|
||||
|
||||
public CodexNode(ILog log, CodexAccess codexAccess, IFileManager fileManager, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks)
|
||||
{
|
||||
this.codexAccess = codexAccess;
|
||||
this.fileManager = fileManager;
|
||||
Marketplace = marketplaceAccess;
|
||||
this.hooks = hooks;
|
||||
Version = new DebugInfoVersion();
|
||||
transferSpeeds = new TransferSpeeds();
|
||||
|
||||
this.log = new LogPrefixer(log, $"{GetName()} ");
|
||||
}
|
||||
|
||||
public void Awake()
|
||||
{
|
||||
hooks.OnNodeStarting(codexAccess.GetStartUtc(), codexAccess.GetImageName(), codexAccess.GetEthAccount());
|
||||
}
|
||||
|
||||
public void Initialize()
|
||||
{
|
||||
InitializePeerNodeId();
|
||||
InitializeLogReplacements();
|
||||
|
||||
hooks.OnNodeStarted(this, peerId, nodeId);
|
||||
}
|
||||
|
||||
public IMarketplaceAccess Marketplace { get; }
|
||||
public DebugInfoVersion Version { get; private set; }
|
||||
public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
|
||||
|
||||
public StoragePurchase? GetPurchaseStatus(string purchaseId)
|
||||
{
|
||||
return codexAccess.GetPurchaseStatus(purchaseId);
|
||||
}
|
||||
|
||||
public EthAddress EthAddress
|
||||
{
|
||||
get
|
||||
{
|
||||
EnsureMarketplace();
|
||||
return codexAccess.GetEthAccount()!.EthAddress;
|
||||
}
|
||||
}
|
||||
|
||||
public EthAccount EthAccount
|
||||
{
|
||||
get
|
||||
{
|
||||
EnsureMarketplace();
|
||||
return codexAccess.GetEthAccount()!;
|
||||
}
|
||||
}
|
||||
|
||||
public string GetName()
|
||||
{
|
||||
return codexAccess.GetName();
|
||||
}
|
||||
|
||||
public string GetImageName()
|
||||
{
|
||||
return codexAccess.GetImageName();
|
||||
}
|
||||
|
||||
public string GetPeerId()
|
||||
{
|
||||
return peerId;
|
||||
}
|
||||
|
||||
public DebugInfo GetDebugInfo(bool log = false)
|
||||
{
|
||||
var debugInfo = codexAccess.GetDebugInfo();
|
||||
if (log)
|
||||
{
|
||||
var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
|
||||
Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
|
||||
}
|
||||
return debugInfo;
|
||||
}
|
||||
|
||||
public void SetLogLevel(string logLevel)
|
||||
{
|
||||
codexAccess.SetLogLevel(logLevel);
|
||||
}
|
||||
|
||||
public string GetSpr()
|
||||
{
|
||||
return codexAccess.GetSpr();
|
||||
}
|
||||
|
||||
public DebugPeer GetDebugPeer(string peerId)
|
||||
{
|
||||
return codexAccess.GetDebugPeer(peerId);
|
||||
}
|
||||
|
||||
public ContentId UploadFile(TrackedFile file)
|
||||
{
|
||||
return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"");
|
||||
}
|
||||
|
||||
public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition)
|
||||
{
|
||||
using var fileStream = File.OpenRead(file.Filename);
|
||||
var uniqueId = Guid.NewGuid().ToString();
|
||||
var size = file.GetFilesize();
|
||||
|
||||
hooks.OnFileUploading(uniqueId, size);
|
||||
|
||||
var input = new UploadInput(contentType, contentDisposition, fileStream);
|
||||
var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
|
||||
var measurement = Stopwatch.Measure(log, logMessage, () =>
|
||||
{
|
||||
return codexAccess.UploadFile(input);
|
||||
});
|
||||
|
||||
var response = measurement.Value;
|
||||
transferSpeeds.AddUploadSample(size, measurement.Duration);
|
||||
|
||||
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
|
||||
if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
|
||||
|
||||
Log($"Uploaded file {file.Describe()}. Received contentId: '{response}'.");
|
||||
|
||||
var cid = new ContentId(response);
|
||||
hooks.OnFileUploaded(uniqueId, size, cid);
|
||||
return cid;
|
||||
}
|
||||
|
||||
public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
|
||||
{
|
||||
var file = fileManager.CreateEmptyFile(fileLabel);
|
||||
hooks.OnFileDownloading(contentId);
|
||||
Log($"Downloading '{contentId}'...");
|
||||
|
||||
var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
|
||||
var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file));
|
||||
|
||||
var size = file.GetFilesize();
|
||||
transferSpeeds.AddDownloadSample(size, measurement);
|
||||
hooks.OnFileDownloaded(size, contentId);
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
public LocalDataset DownloadStreamless(ContentId cid)
|
||||
{
|
||||
Log($"Downloading streamless '{cid}' (no-wait)");
|
||||
return codexAccess.DownloadStreamless(cid);
|
||||
}
|
||||
|
||||
public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
|
||||
{
|
||||
Log($"Downloading streamless '{cid}' (wait till finished)");
|
||||
|
||||
var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
|
||||
{
|
||||
var startSpace = Space();
|
||||
var result = codexAccess.DownloadStreamless(cid);
|
||||
WaitUntilQuotaUsedIncreased(startSpace, size);
|
||||
return result;
|
||||
});
|
||||
|
||||
return sw.Value;
|
||||
}
|
||||
|
||||
public LocalDataset DownloadManifestOnly(ContentId cid)
|
||||
{
|
||||
Log($"Downloading manifest-only '{cid}'");
|
||||
return codexAccess.DownloadManifestOnly(cid);
|
||||
}
|
||||
|
||||
public LocalDatasetList LocalFiles()
|
||||
{
|
||||
return codexAccess.LocalFiles();
|
||||
}
|
||||
|
||||
public CodexSpace Space()
|
||||
{
|
||||
return codexAccess.Space();
|
||||
}
|
||||
|
||||
public void ConnectToPeer(ICodexNode node)
|
||||
{
|
||||
var peer = (CodexNode)node;
|
||||
|
||||
Log($"Connecting to peer {peer.GetName()}...");
|
||||
var peerInfo = node.GetDebugInfo();
|
||||
codexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
|
||||
|
||||
Log($"Successfully connected to peer {peer.GetName()}.");
|
||||
}
|
||||
|
||||
public void DeleteDataDirFolder()
|
||||
{
|
||||
codexAccess.DeleteDataDirFolder();
|
||||
}
|
||||
|
||||
public void Stop(bool waitTillStopped)
|
||||
{
|
||||
Log("Stopping...");
|
||||
hooks.OnNodeStopping();
|
||||
codexAccess.Stop(waitTillStopped);
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(string additionalName = "")
|
||||
{
|
||||
return codexAccess.DownloadLog(additionalName);
|
||||
}
|
||||
|
||||
public Address GetDiscoveryEndpoint()
|
||||
{
|
||||
return codexAccess.GetDiscoveryEndpoint();
|
||||
}
|
||||
|
||||
public Address GetApiEndpoint()
|
||||
{
|
||||
return codexAccess.GetApiEndpoint();
|
||||
}
|
||||
|
||||
public Address GetListenEndpoint()
|
||||
{
|
||||
return codexAccess.GetListenEndpoint();
|
||||
}
|
||||
|
||||
public Address GetMetricsScrapeTarget()
|
||||
{
|
||||
var address = codexAccess.GetMetricsEndpoint();
|
||||
if (address == null) throw new Exception("Metrics ScrapeTarget accessed, but node was not started with EnableMetrics()");
|
||||
return address;
|
||||
}
|
||||
|
||||
public bool HasCrashed()
|
||||
{
|
||||
return codexAccess.HasCrashed();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"CodexNode:{GetName()}";
|
||||
}
|
||||
|
||||
private void InitializePeerNodeId()
|
||||
{
|
||||
var debugInfo = Time.Retry(codexAccess.GetDebugInfo, "ensure online");
|
||||
if (!debugInfo.Version.IsValid())
|
||||
{
|
||||
throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
|
||||
}
|
||||
|
||||
peerId = debugInfo.Id;
|
||||
nodeId = debugInfo.Table.LocalNode.NodeId;
|
||||
Version = debugInfo.Version;
|
||||
}
|
||||
|
||||
private void InitializeLogReplacements()
|
||||
{
|
||||
var nodeName = GetName();
|
||||
|
||||
log.AddStringReplace(peerId, nodeName);
|
||||
log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
|
||||
log.AddStringReplace(nodeId, nodeName);
|
||||
log.AddStringReplace(CodexUtils.ToShortId(nodeId), nodeName);
|
||||
}
|
||||
|
||||
private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
|
||||
{
|
||||
var peerId = peer.GetDiscoveryEndpoint().Host
|
||||
.Replace("http://", "")
|
||||
.Replace("https://", "");
|
||||
|
||||
return peerInfo.Addrs.Select(a => a
|
||||
.Replace("0.0.0.0", peerId))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private void DownloadToFile(string contentId, TrackedFile file)
|
||||
{
|
||||
using var fileStream = File.OpenWrite(file.Filename);
|
||||
var timeout = TimeSpan.FromMinutes(2.0); // todo: make this user-controllable.
|
||||
try
|
||||
{
|
||||
// Type of stream generated by openAPI client does not support timeouts.
|
||||
var start = DateTime.UtcNow;
|
||||
var cts = new CancellationTokenSource();
|
||||
var downloadTask = Task.Run(() =>
|
||||
{
|
||||
using var downloadStream = codexAccess.DownloadFile(contentId);
|
||||
downloadStream.CopyTo(fileStream);
|
||||
}, cts.Token);
|
||||
|
||||
while (DateTime.UtcNow - start < timeout)
|
||||
{
|
||||
if (downloadTask.IsFaulted) throw downloadTask.Exception;
|
||||
if (downloadTask.IsCompletedSuccessfully) return;
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
|
||||
cts.Cancel();
|
||||
throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log($"Failed to download file '{contentId}': {ex}");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
|
||||
{
|
||||
WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
|
||||
}
|
||||
|
||||
public void WaitUntilQuotaUsedIncreased(
|
||||
CodexSpace startSpace,
|
||||
ByteSize expectedIncreaseOfQuotaUsed,
|
||||
TimeSpan maxTimeout)
|
||||
{
|
||||
Log($"Waiting until quotaUsed " +
|
||||
$"(start: {startSpace.QuotaUsedBytes}) " +
|
||||
$"increases by {expectedIncreaseOfQuotaUsed} " +
|
||||
$"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
|
||||
|
||||
var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
|
||||
maxTimeout: maxTimeout,
|
||||
sleepAfterFail: TimeSpan.FromSeconds(3),
|
||||
onFail: f => { });
|
||||
|
||||
retry.Run(() =>
|
||||
{
|
||||
var space = Space();
|
||||
var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
|
||||
|
||||
if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
|
||||
throw new Exception($"Expected quota-used not reached. " +
|
||||
$"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
|
||||
$"Actual increase: {increase} " +
|
||||
$"Actual used: {space.QuotaUsedBytes}");
|
||||
});
|
||||
}
|
||||
|
||||
private void EnsureMarketplace()
|
||||
{
|
||||
if (codexAccess.GetEthAccount() == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
log.Log(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using CodexClient.Hooks;
|
||||
using FileUtils;
|
||||
using Logging;
|
||||
using WebUtils;
|
||||
|
||||
namespace CodexClient
|
||||
{
|
||||
public class CodexNodeFactory
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly IFileManager fileManager;
|
||||
private readonly CodexHooksFactory hooksFactory;
|
||||
private readonly IHttpFactory httpFactory;
|
||||
private readonly IProcessControlFactory processControlFactory;
|
||||
|
||||
public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory)
|
||||
{
|
||||
this.log = log;
|
||||
this.fileManager = fileManager;
|
||||
this.hooksFactory = hooksFactory;
|
||||
this.httpFactory = httpFactory;
|
||||
this.processControlFactory = processControlFactory;
|
||||
}
|
||||
|
||||
public CodexNodeFactory(ILog log, HttpFactory httpFactory, string dataDir)
|
||||
: this(log, new FileManager(log, dataDir), new CodexHooksFactory(), httpFactory, new DoNothingProcessControlFactory())
|
||||
{
|
||||
}
|
||||
|
||||
public CodexNodeFactory(ILog log, string dataDir)
|
||||
: this(log, new HttpFactory(log), dataDir)
|
||||
{
|
||||
}
|
||||
|
||||
public ICodexNode CreateCodexNode(ICodexInstance instance)
|
||||
{
|
||||
var processControl = processControlFactory.CreateProcessControl(instance);
|
||||
var access = new CodexAccess(log, httpFactory, processControl, instance);
|
||||
var hooks = hooksFactory.CreateHooks(access.GetName());
|
||||
var marketplaceAccess = CreateMarketplaceAccess(instance, access, hooks);
|
||||
var node = new CodexNode(log, access, fileManager, marketplaceAccess, hooks);
|
||||
node.Initialize();
|
||||
return node;
|
||||
}
|
||||
|
||||
private IMarketplaceAccess CreateMarketplaceAccess(ICodexInstance instance, CodexAccess access, ICodexNodeHooks hooks)
|
||||
{
|
||||
if (instance.EthAccount == null) return new MarketplaceUnavailable();
|
||||
return new MarketplaceAccess(log, access, hooks);
|
||||
}
|
||||
}
|
||||
}
|
||||
+11
-1
@@ -1,7 +1,7 @@
|
||||
using Newtonsoft.Json;
|
||||
using Utils;
|
||||
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public class DebugInfo
|
||||
{
|
||||
@@ -109,6 +109,16 @@ namespace CodexPlugin
|
||||
{
|
||||
return HashCode.Combine(Id);
|
||||
}
|
||||
|
||||
public static bool operator ==(ContentId a, ContentId b)
|
||||
{
|
||||
return a.Id == b.Id;
|
||||
}
|
||||
|
||||
public static bool operator !=(ContentId a, ContentId b)
|
||||
{
|
||||
return a.Id != b.Id;
|
||||
}
|
||||
}
|
||||
|
||||
public class CodexSpace
|
||||
@@ -1,4 +1,4 @@
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public static class CodexUtils
|
||||
{
|
||||
+8
-6
@@ -1,7 +1,6 @@
|
||||
using GethPlugin;
|
||||
using Utils;
|
||||
using Utils;
|
||||
|
||||
namespace CodexPlugin.Hooks
|
||||
namespace CodexClient.Hooks
|
||||
{
|
||||
public interface ICodexHooksProvider
|
||||
{
|
||||
@@ -10,11 +9,14 @@ namespace CodexPlugin.Hooks
|
||||
|
||||
public class CodexHooksFactory
|
||||
{
|
||||
public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
|
||||
public List<ICodexHooksProvider> Providers { get; } = new List<ICodexHooksProvider>();
|
||||
|
||||
public ICodexNodeHooks CreateHooks(string nodeName)
|
||||
{
|
||||
return Provider.CreateHooks(nodeName);
|
||||
if (Providers.Count == 0) return new DoNothingCodexHooks();
|
||||
|
||||
var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray();
|
||||
return new MuxingCodexNodeHooks(hooks);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +46,7 @@ namespace CodexPlugin.Hooks
|
||||
{
|
||||
}
|
||||
|
||||
public void OnNodeStarted(string peerId, string nodeId)
|
||||
public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using Utils;
|
||||
|
||||
namespace CodexClient.Hooks
|
||||
{
|
||||
public interface ICodexNodeHooks
|
||||
{
|
||||
void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
|
||||
void OnNodeStarted(ICodexNode node, string peerId, string nodeId);
|
||||
void OnNodeStopping();
|
||||
void OnFileUploading(string uid, ByteSize size);
|
||||
void OnFileUploaded(string uid, ByteSize size, ContentId cid);
|
||||
void OnFileDownloading(ContentId cid);
|
||||
void OnFileDownloaded(ByteSize size, ContentId cid);
|
||||
void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract);
|
||||
void OnStorageContractUpdated(StoragePurchase purchaseStatus);
|
||||
void OnStorageAvailabilityCreated(StorageAvailability response);
|
||||
}
|
||||
|
||||
public class MuxingCodexNodeHooks : ICodexNodeHooks
|
||||
{
|
||||
private readonly ICodexNodeHooks[] backingHooks;
|
||||
|
||||
public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks)
|
||||
{
|
||||
this.backingHooks = backingHooks;
|
||||
}
|
||||
|
||||
public void OnFileDownloaded(ByteSize size, ContentId cid)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnFileDownloaded(size, cid);
|
||||
}
|
||||
|
||||
public void OnFileDownloading(ContentId cid)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnFileDownloading(cid);
|
||||
}
|
||||
|
||||
public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnFileUploaded(uid, size, cid);
|
||||
}
|
||||
|
||||
public void OnFileUploading(string uid, ByteSize size)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnFileUploading(uid, size);
|
||||
}
|
||||
|
||||
public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId);
|
||||
}
|
||||
|
||||
public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnNodeStarting(startUtc, image, ethAccount);
|
||||
}
|
||||
|
||||
public void OnNodeStopping()
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnNodeStopping();
|
||||
}
|
||||
|
||||
public void OnStorageAvailabilityCreated(StorageAvailability response)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnStorageAvailabilityCreated(response);
|
||||
}
|
||||
|
||||
public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnStorageContractSubmitted(storagePurchaseContract);
|
||||
}
|
||||
|
||||
public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
|
||||
{
|
||||
foreach (var h in backingHooks) h.OnStorageContractUpdated(purchaseStatus);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
using CodexOpenApi;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System.Numerics;
|
||||
using Utils;
|
||||
|
||||
namespace CodexClient
|
||||
{
|
||||
public class Mapper
|
||||
{
|
||||
public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo)
|
||||
{
|
||||
return new DebugInfo
|
||||
{
|
||||
Id = debugInfo.Id,
|
||||
Spr = debugInfo.Spr,
|
||||
Addrs = debugInfo.Addrs.ToArray(),
|
||||
AnnounceAddresses = debugInfo.AnnounceAddresses.ToArray(),
|
||||
Version = Map(debugInfo.Codex),
|
||||
Table = Map(debugInfo.Table)
|
||||
};
|
||||
}
|
||||
|
||||
public LocalDatasetList Map(CodexOpenApi.DataList dataList)
|
||||
{
|
||||
return new LocalDatasetList
|
||||
{
|
||||
Content = dataList.Content.Select(Map).ToArray()
|
||||
};
|
||||
}
|
||||
|
||||
public LocalDataset Map(CodexOpenApi.DataItem dataItem)
|
||||
{
|
||||
return new LocalDataset
|
||||
{
|
||||
Cid = new ContentId(dataItem.Cid),
|
||||
Manifest = MapManifest(dataItem.Manifest)
|
||||
};
|
||||
}
|
||||
|
||||
public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
|
||||
{
|
||||
return new CodexOpenApi.SalesAvailabilityCREATE
|
||||
{
|
||||
Duration = ToDecInt(availability.MaxDuration.TotalSeconds),
|
||||
MinPricePerBytePerSecond = ToDecInt(availability.MinPricePerBytePerSecond),
|
||||
TotalCollateral = ToDecInt(availability.TotalCollateral),
|
||||
TotalSize = ToDecInt(availability.TotalSpace.SizeInBytes)
|
||||
};
|
||||
}
|
||||
|
||||
public CodexOpenApi.StorageRequestCreation Map(StoragePurchaseRequest purchase)
|
||||
{
|
||||
return new CodexOpenApi.StorageRequestCreation
|
||||
{
|
||||
Duration = ToDecInt(purchase.Duration.TotalSeconds),
|
||||
ProofProbability = ToDecInt(purchase.ProofProbability),
|
||||
PricePerBytePerSecond = ToDecInt(purchase.PricePerBytePerSecond),
|
||||
CollateralPerByte = ToDecInt(purchase.CollateralPerByte),
|
||||
Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
|
||||
Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
|
||||
Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
|
||||
};
|
||||
}
|
||||
|
||||
public StorageAvailability[] Map(ICollection<CodexOpenApi.SalesAvailabilityREAD> availabilities)
|
||||
{
|
||||
return availabilities.Select(a => Map(a)).ToArray();
|
||||
}
|
||||
|
||||
public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD availability)
|
||||
{
|
||||
return new StorageAvailability
|
||||
(
|
||||
ToByteSize(availability.TotalSize),
|
||||
ToTimespan(availability.Duration),
|
||||
new TestToken(ToBigIng(availability.MinPricePerBytePerSecond)),
|
||||
new TestToken(ToBigIng(availability.TotalCollateral))
|
||||
)
|
||||
{
|
||||
Id = availability.Id,
|
||||
FreeSpace = ToByteSize(availability.FreeSize),
|
||||
};
|
||||
}
|
||||
|
||||
public StoragePurchase Map(CodexOpenApi.Purchase purchase)
|
||||
{
|
||||
return new StoragePurchase
|
||||
{
|
||||
Request = Map(purchase.Request),
|
||||
State = purchase.State.ToString(), //Map(purchase.State),
|
||||
Error = purchase.Error
|
||||
};
|
||||
}
|
||||
|
||||
public StoragePurchaseState Map(PurchaseState purchaseState)
|
||||
{
|
||||
// TODO: to be re-enabled when marketplace api lines up with openapi.yaml.
|
||||
|
||||
// Explicit mapping: If the API changes, we will get compile errors here.
|
||||
// That's what we want.
|
||||
switch (purchaseState)
|
||||
{
|
||||
case PurchaseState.Cancelled:
|
||||
return StoragePurchaseState.Cancelled;
|
||||
case PurchaseState.Error:
|
||||
return StoragePurchaseState.Error;
|
||||
case PurchaseState.Failed:
|
||||
return StoragePurchaseState.Failed;
|
||||
case PurchaseState.Finished:
|
||||
return StoragePurchaseState.Finished;
|
||||
case PurchaseState.Pending:
|
||||
return StoragePurchaseState.Pending;
|
||||
case PurchaseState.Started:
|
||||
return StoragePurchaseState.Started;
|
||||
case PurchaseState.Submitted:
|
||||
return StoragePurchaseState.Submitted;
|
||||
case PurchaseState.Unknown:
|
||||
return StoragePurchaseState.Unknown;
|
||||
}
|
||||
|
||||
throw new Exception("API incompatibility detected. Unknown purchaseState: " + purchaseState.ToString());
|
||||
}
|
||||
|
||||
public StorageRequest Map(CodexOpenApi.StorageRequest request)
|
||||
{
|
||||
return new StorageRequest
|
||||
{
|
||||
Ask = Map(request.Ask),
|
||||
Content = Map(request.Content),
|
||||
Id = request.Id,
|
||||
Client = request.Client,
|
||||
Expiry = request.Expiry,
|
||||
Nonce = request.Nonce
|
||||
};
|
||||
}
|
||||
|
||||
public StorageAsk Map(CodexOpenApi.StorageAsk ask)
|
||||
{
|
||||
return new StorageAsk
|
||||
{
|
||||
Duration = ask.Duration,
|
||||
MaxSlotLoss = ask.MaxSlotLoss,
|
||||
ProofProbability = ask.ProofProbability,
|
||||
PricePerBytePerSecond = ask.PricePerBytePerSecond,
|
||||
Slots = ask.Slots,
|
||||
SlotSize = ask.SlotSize
|
||||
};
|
||||
}
|
||||
|
||||
public StorageContent Map(CodexOpenApi.Content content)
|
||||
{
|
||||
return new StorageContent
|
||||
{
|
||||
Cid = content.Cid
|
||||
};
|
||||
}
|
||||
|
||||
public CodexSpace Map(CodexOpenApi.Space space)
|
||||
{
|
||||
return new CodexSpace
|
||||
{
|
||||
QuotaMaxBytes = space.QuotaMaxBytes,
|
||||
QuotaReservedBytes = space.QuotaReservedBytes,
|
||||
QuotaUsedBytes = space.QuotaUsedBytes,
|
||||
TotalBlocks = space.TotalBlocks
|
||||
};
|
||||
}
|
||||
|
||||
private DebugInfoVersion Map(CodexOpenApi.CodexVersion obj)
|
||||
{
|
||||
return new DebugInfoVersion
|
||||
{
|
||||
Version = obj.Version,
|
||||
Revision = obj.Revision
|
||||
};
|
||||
}
|
||||
|
||||
private DebugInfoTable Map(CodexOpenApi.PeersTable obj)
|
||||
{
|
||||
return new DebugInfoTable
|
||||
{
|
||||
LocalNode = Map(obj.LocalNode),
|
||||
Nodes = Map(obj.Nodes)
|
||||
};
|
||||
}
|
||||
|
||||
private DebugInfoTableNode Map(CodexOpenApi.Node? token)
|
||||
{
|
||||
if (token == null) return new DebugInfoTableNode();
|
||||
return new DebugInfoTableNode
|
||||
{
|
||||
Address = token.Address,
|
||||
NodeId = token.NodeId,
|
||||
PeerId = token.PeerId,
|
||||
Record = token.Record,
|
||||
Seen = token.Seen
|
||||
};
|
||||
}
|
||||
|
||||
private DebugInfoTableNode[] Map(ICollection<CodexOpenApi.Node> nodes)
|
||||
{
|
||||
if (nodes == null || nodes.Count == 0)
|
||||
{
|
||||
return new DebugInfoTableNode[0];
|
||||
}
|
||||
|
||||
return nodes.Select(Map).ToArray();
|
||||
}
|
||||
|
||||
private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
|
||||
{
|
||||
return new Manifest
|
||||
{
|
||||
BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
|
||||
OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
|
||||
RootHash = manifest.TreeCid,
|
||||
Protected = manifest.Protected
|
||||
};
|
||||
}
|
||||
|
||||
private JArray JArray(IDictionary<string, object> map, string name)
|
||||
{
|
||||
return (JArray)map[name];
|
||||
}
|
||||
|
||||
private JObject JObject(IDictionary<string, object> map, string name)
|
||||
{
|
||||
return (JObject)map[name];
|
||||
}
|
||||
|
||||
private string StringOrEmpty(JObject obj, string name)
|
||||
{
|
||||
if (obj.TryGetValue(name, out var token))
|
||||
{
|
||||
var str = (string?)token;
|
||||
if (!string.IsNullOrEmpty(str)) return str;
|
||||
}
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private bool Bool(JObject obj, string name)
|
||||
{
|
||||
if (obj.TryGetValue(name, out var token))
|
||||
{
|
||||
return (bool)token;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private string ToDecInt(double d)
|
||||
{
|
||||
var i = new BigInteger(d);
|
||||
return i.ToString("D");
|
||||
}
|
||||
|
||||
private string ToDecInt(TestToken t)
|
||||
{
|
||||
return t.TstWei.ToString("D");
|
||||
}
|
||||
|
||||
private BigInteger ToBigIng(string tokens)
|
||||
{
|
||||
return BigInteger.Parse(tokens);
|
||||
}
|
||||
|
||||
private TimeSpan ToTimespan(string duration)
|
||||
{
|
||||
return TimeSpan.FromSeconds(Convert.ToInt32(duration));
|
||||
}
|
||||
|
||||
private ByteSize ToByteSize(string size)
|
||||
{
|
||||
return new ByteSize(Convert.ToInt64(size));
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
-10
@@ -1,12 +1,13 @@
|
||||
using CodexPlugin.Hooks;
|
||||
using CodexClient.Hooks;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public interface IMarketplaceAccess
|
||||
{
|
||||
string MakeStorageAvailable(StorageAvailability availability);
|
||||
StorageAvailability[] GetAvailabilities();
|
||||
IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
|
||||
}
|
||||
|
||||
@@ -26,8 +27,12 @@ namespace CodexPlugin
|
||||
public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
|
||||
{
|
||||
purchase.Log(log);
|
||||
var swResult = Stopwatch.Measure(log, nameof(RequestStorage), () =>
|
||||
{
|
||||
return codexAccess.RequestStorage(purchase);
|
||||
});
|
||||
|
||||
var response = codexAccess.RequestStorage(purchase);
|
||||
var response = swResult.Value;
|
||||
|
||||
if (string.IsNullOrEmpty(response) ||
|
||||
response == "Unable to encode manifest" ||
|
||||
@@ -41,12 +46,7 @@ namespace CodexPlugin
|
||||
|
||||
Log($"Storage requested successfully. PurchaseId: '{response}'.");
|
||||
|
||||
var contract = new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
|
||||
contract.WaitForStorageContractSubmitted();
|
||||
|
||||
hooks.OnStorageContractSubmitted(contract);
|
||||
|
||||
return contract;
|
||||
return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
|
||||
}
|
||||
|
||||
public string MakeStorageAvailable(StorageAvailability availability)
|
||||
@@ -61,9 +61,17 @@ namespace CodexPlugin
|
||||
return response.Id;
|
||||
}
|
||||
|
||||
public StorageAvailability[] GetAvailabilities()
|
||||
{
|
||||
var result = codexAccess.GetAvailabilities();
|
||||
Log($"Got {result.Length} availabilities:");
|
||||
foreach (var a in result) a.Log(log);
|
||||
return result;
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
|
||||
log.Log($"{codexAccess.GetName()} {msg}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,6 +89,12 @@ namespace CodexPlugin
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public StorageAvailability[] GetAvailabilities()
|
||||
{
|
||||
Unavailable();
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
private void Unavailable()
|
||||
{
|
||||
FrameworkAssert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");
|
||||
+34
-18
@@ -1,10 +1,7 @@
|
||||
using CodexContractsPlugin;
|
||||
using CodexOpenApi;
|
||||
using Logging;
|
||||
using System.Data;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public class StoragePurchaseRequest
|
||||
{
|
||||
@@ -14,8 +11,8 @@ namespace CodexPlugin
|
||||
}
|
||||
|
||||
public ContentId ContentId { get; set; }
|
||||
public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
|
||||
public TestToken RequiredCollateral { get; set; } = 1.TstWei();
|
||||
public TestToken PricePerBytePerSecond { get; set; } = 1.TstWei();
|
||||
public TestToken CollateralPerByte { get; set; } = 1.TstWei();
|
||||
public uint MinRequiredNumberOfNodes { get; set; }
|
||||
public uint NodeFailureTolerance { get; set; }
|
||||
public int ProofProbability { get; set; }
|
||||
@@ -25,8 +22,8 @@ namespace CodexPlugin
|
||||
public void Log(ILog log)
|
||||
{
|
||||
log.Log($"Requesting storage for: {ContentId.Id}... (" +
|
||||
$"pricePerSlotPerSecond: {PricePerSlotPerSecond}, " +
|
||||
$"requiredCollateral: {RequiredCollateral}, " +
|
||||
$"pricePerBytePerSecond: {PricePerBytePerSecond}, " +
|
||||
$"collateralPerByte: {CollateralPerByte}, " +
|
||||
$"minRequiredNumberOfNodes: {MinRequiredNumberOfNodes}, " +
|
||||
$"nodeFailureTolerance: {NodeFailureTolerance}, " +
|
||||
$"proofProbability: {ProofProbability}, " +
|
||||
@@ -40,6 +37,24 @@ namespace CodexPlugin
|
||||
public string State { get; set; } = string.Empty;
|
||||
public string Error { get; set; } = string.Empty;
|
||||
public StorageRequest Request { get; set; } = null!;
|
||||
|
||||
public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
|
||||
public bool IsError => State.ToLowerInvariant().Contains("error");
|
||||
public bool IsFinished => State.ToLowerInvariant().Contains("finished");
|
||||
public bool IsStarted => State.ToLowerInvariant().Contains("started");
|
||||
public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
|
||||
}
|
||||
|
||||
public enum StoragePurchaseState
|
||||
{
|
||||
Cancelled = 0,
|
||||
Error = 1,
|
||||
Failed = 2,
|
||||
Finished = 3,
|
||||
Pending = 4,
|
||||
Started = 5,
|
||||
Submitted = 6,
|
||||
Unknown = 7,
|
||||
}
|
||||
|
||||
public class StorageRequest
|
||||
@@ -58,7 +73,7 @@ namespace CodexPlugin
|
||||
public string SlotSize { get; set; } = string.Empty;
|
||||
public string Duration { get; set; } = string.Empty;
|
||||
public string ProofProbability { get; set; } = string.Empty;
|
||||
public string Reward { get; set; } = string.Empty;
|
||||
public string PricePerBytePerSecond { get; set; } = string.Empty;
|
||||
public int MaxSlotLoss { get; set; }
|
||||
}
|
||||
|
||||
@@ -71,27 +86,28 @@ namespace CodexPlugin
|
||||
|
||||
public class StorageAvailability
|
||||
{
|
||||
public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPriceForTotalSpace, TestToken maxCollateral)
|
||||
public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
|
||||
{
|
||||
TotalSpace = totalSpace;
|
||||
MaxDuration = maxDuration;
|
||||
MinPriceForTotalSpace = minPriceForTotalSpace;
|
||||
MaxCollateral = maxCollateral;
|
||||
MinPricePerBytePerSecond = minPricePerBytePerSecond;
|
||||
TotalCollateral = totalCollateral;
|
||||
}
|
||||
|
||||
public string Id { get; set; } = string.Empty;
|
||||
public ByteSize TotalSpace { get; }
|
||||
public TimeSpan MaxDuration { get; }
|
||||
public TestToken MinPriceForTotalSpace { get; }
|
||||
public TestToken MaxCollateral { get; }
|
||||
public TestToken MinPricePerBytePerSecond { get; }
|
||||
public TestToken TotalCollateral { get; }
|
||||
public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
|
||||
|
||||
public void Log(ILog log)
|
||||
{
|
||||
log.Log($"Making storage available... (" +
|
||||
log.Log($"Storage Availability: (" +
|
||||
$"totalSize: {TotalSpace}, " +
|
||||
$"maxDuration: {Time.FormatDuration(MaxDuration)}, " +
|
||||
$"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +
|
||||
$"maxCollateral: {MaxCollateral})");
|
||||
$"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
|
||||
$"totalCollateral: {TotalCollateral})");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using Logging;
|
||||
|
||||
namespace CodexClient
|
||||
{
|
||||
public interface IProcessControlFactory
|
||||
{
|
||||
IProcessControl CreateProcessControl(ICodexInstance instance);
|
||||
}
|
||||
|
||||
public interface IProcessControl
|
||||
{
|
||||
void Stop(bool waitTillStopped);
|
||||
IDownloadedLog DownloadLog(LogFile file);
|
||||
void DeleteDataDirFolder();
|
||||
bool HasCrashed();
|
||||
}
|
||||
|
||||
public class DoNothingProcessControlFactory : IProcessControlFactory
|
||||
{
|
||||
public IProcessControl CreateProcessControl(ICodexInstance instance)
|
||||
{
|
||||
return new DoNothingProcessControl();
|
||||
}
|
||||
}
|
||||
|
||||
public class DoNothingProcessControl : IProcessControl
|
||||
{
|
||||
public void DeleteDataDirFolder()
|
||||
{
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(LogFile file)
|
||||
{
|
||||
throw new NotImplementedException("Not supported by DoNothingProcessControl");
|
||||
}
|
||||
|
||||
public bool HasCrashed()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
public void Stop(bool waitTillStopped)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
+38
-10
@@ -1,18 +1,20 @@
|
||||
using CodexPlugin.Hooks;
|
||||
using CodexClient.Hooks;
|
||||
using Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Utils;
|
||||
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public interface IStoragePurchaseContract
|
||||
{
|
||||
string PurchaseId { get; }
|
||||
StoragePurchaseRequest Purchase { get; }
|
||||
ContentId ContentId { get; }
|
||||
StoragePurchase? GetStatus();
|
||||
void WaitForStorageContractSubmitted();
|
||||
void WaitForStorageContractStarted();
|
||||
void WaitForStorageContractFinished();
|
||||
void WaitForContractFailed();
|
||||
}
|
||||
|
||||
public class StoragePurchaseContract : IStoragePurchaseContract
|
||||
@@ -20,11 +22,13 @@ namespace CodexPlugin
|
||||
private readonly ILog log;
|
||||
private readonly CodexAccess codexAccess;
|
||||
private readonly ICodexNodeHooks hooks;
|
||||
private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
|
||||
private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(60);
|
||||
private readonly DateTime contractPendingUtc = DateTime.UtcNow;
|
||||
private DateTime? contractSubmittedUtc = DateTime.UtcNow;
|
||||
private DateTime? contractStartedUtc;
|
||||
private DateTime? contractFinishedUtc;
|
||||
private string lastState = string.Empty;
|
||||
private ContentId encodedContentId = new ContentId();
|
||||
|
||||
public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase, ICodexNodeHooks hooks)
|
||||
{
|
||||
@@ -33,21 +37,40 @@ namespace CodexPlugin
|
||||
PurchaseId = purchaseId;
|
||||
Purchase = purchase;
|
||||
this.hooks = hooks;
|
||||
ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
|
||||
}
|
||||
|
||||
public string PurchaseId { get; }
|
||||
public StoragePurchaseRequest Purchase { get; }
|
||||
public ContentId ContentId { get; }
|
||||
public ContentId ContentId
|
||||
{
|
||||
get
|
||||
{
|
||||
if (string.IsNullOrEmpty(encodedContentId.Id)) GetStatus();
|
||||
return encodedContentId;
|
||||
}
|
||||
}
|
||||
|
||||
public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
|
||||
public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
|
||||
public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
|
||||
|
||||
public StoragePurchase? GetStatus()
|
||||
{
|
||||
var status = codexAccess.GetPurchaseStatus(PurchaseId);
|
||||
if (status != null)
|
||||
{
|
||||
encodedContentId = new ContentId(status.Request.Content.Cid);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
public void WaitForStorageContractSubmitted()
|
||||
{
|
||||
WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
|
||||
var timeout = Purchase.Expiry + gracePeriod;
|
||||
var raiseHook = lastState != "submitted";
|
||||
WaitForStorageContractState(timeout, "submitted", sleep: 200);
|
||||
contractSubmittedUtc = DateTime.UtcNow;
|
||||
if (raiseHook) hooks.OnStorageContractSubmitted(this);
|
||||
LogSubmittedDuration();
|
||||
AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
|
||||
}
|
||||
@@ -76,14 +99,19 @@ namespace CodexPlugin
|
||||
AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
|
||||
}
|
||||
|
||||
public StoragePurchase GetPurchaseStatus(string purchaseId)
|
||||
public void WaitForContractFailed()
|
||||
{
|
||||
return codexAccess.GetPurchaseStatus(purchaseId);
|
||||
if (!contractStartedUtc.HasValue)
|
||||
{
|
||||
WaitForStorageContractStarted();
|
||||
}
|
||||
var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
|
||||
var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
|
||||
WaitForStorageContractState(timeout, "failed");
|
||||
}
|
||||
|
||||
private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
|
||||
{
|
||||
var lastState = "";
|
||||
var waitStart = DateTime.UtcNow;
|
||||
|
||||
Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
|
||||
@@ -115,7 +143,7 @@ namespace CodexPlugin
|
||||
private void LogSubmittedDuration()
|
||||
{
|
||||
Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
|
||||
$"( < {Time.FormatDuration(gracePeriod)})");
|
||||
$"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
|
||||
}
|
||||
|
||||
private void LogStartedDuration()
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
using Utils;
|
||||
|
||||
namespace CodexPlugin
|
||||
namespace CodexClient
|
||||
{
|
||||
public interface ITransferSpeeds
|
||||
{
|
||||
+231
-87
@@ -6,7 +6,7 @@ info:
|
||||
description: "List of endpoints and interfaces available to Codex API users"
|
||||
|
||||
security:
|
||||
- { }
|
||||
- {}
|
||||
|
||||
components:
|
||||
schemas:
|
||||
@@ -23,6 +23,8 @@ components:
|
||||
Id:
|
||||
type: string
|
||||
description: 32bits identifier encoded in hex-decimal string.
|
||||
minLength: 66
|
||||
maxLength: 66
|
||||
example: 0x...
|
||||
|
||||
BigInt:
|
||||
@@ -48,9 +50,9 @@ components:
|
||||
type: string
|
||||
description: Address of Ethereum address
|
||||
|
||||
Reward:
|
||||
PricePerBytePerSecond:
|
||||
type: string
|
||||
description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay
|
||||
description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay
|
||||
|
||||
Duration:
|
||||
type: string
|
||||
@@ -81,33 +83,46 @@ components:
|
||||
id:
|
||||
$ref: "#/components/schemas/PeerId"
|
||||
|
||||
ErasureParameters:
|
||||
type: object
|
||||
properties:
|
||||
totalChunks:
|
||||
type: integer
|
||||
|
||||
PoRParameters:
|
||||
description: Parameters for Proof of Retrievability
|
||||
type: object
|
||||
properties:
|
||||
u:
|
||||
type: string
|
||||
publicKey:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
|
||||
Content:
|
||||
type: object
|
||||
description: Parameters specifying the content
|
||||
properties:
|
||||
cid:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
erasure:
|
||||
$ref: "#/components/schemas/ErasureParameters"
|
||||
por:
|
||||
$ref: "#/components/schemas/PoRParameters"
|
||||
|
||||
Node:
|
||||
type: object
|
||||
properties:
|
||||
nodeId:
|
||||
type: string
|
||||
peerId:
|
||||
type: string
|
||||
record:
|
||||
type: string
|
||||
address:
|
||||
type: string
|
||||
seen:
|
||||
type: boolean
|
||||
|
||||
CodexVersion:
|
||||
type: object
|
||||
properties:
|
||||
version:
|
||||
type: string
|
||||
example: v0.1.7
|
||||
revision:
|
||||
type: string
|
||||
example: 0c647d8
|
||||
|
||||
PeersTable:
|
||||
type: object
|
||||
properties:
|
||||
localNode:
|
||||
$ref: "#/components/schemas/Node"
|
||||
nodes:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/Node"
|
||||
|
||||
DebugInfo:
|
||||
type: object
|
||||
@@ -123,6 +138,14 @@ components:
|
||||
description: Path of the data repository where all nodes data are stored
|
||||
spr:
|
||||
$ref: "#/components/schemas/SPR"
|
||||
announceAddresses:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/MultiAddress"
|
||||
table:
|
||||
$ref: "#/components/schemas/PeersTable"
|
||||
codex:
|
||||
$ref: "#/components/schemas/CodexVersion"
|
||||
|
||||
SalesAvailability:
|
||||
type: object
|
||||
@@ -134,12 +157,12 @@ components:
|
||||
description: Total size of availability's storage in bytes as decimal string
|
||||
duration:
|
||||
$ref: "#/components/schemas/Duration"
|
||||
minPrice:
|
||||
minPricePerBytePerSecond:
|
||||
type: string
|
||||
description: Minimum price to be paid (in amount of tokens) as decimal string
|
||||
maxCollateral:
|
||||
description: Minimal price per byte per second paid (in amount of tokens) for the hosted request's slot for the request's duration as decimal string
|
||||
totalCollateral:
|
||||
type: string
|
||||
description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
|
||||
description: Total collateral (in amount of tokens) that can be used for matching requests
|
||||
|
||||
SalesAvailabilityREAD:
|
||||
allOf:
|
||||
@@ -155,8 +178,8 @@ components:
|
||||
- $ref: "#/components/schemas/SalesAvailability"
|
||||
- required:
|
||||
- totalSize
|
||||
- minPrice
|
||||
- maxCollateral
|
||||
- minPricePerBytePerSecond
|
||||
- totalCollateral
|
||||
- duration
|
||||
|
||||
Slot:
|
||||
@@ -168,7 +191,39 @@ components:
|
||||
$ref: "#/components/schemas/StorageRequest"
|
||||
slotIndex:
|
||||
type: string
|
||||
description: Slot Index as hexadecimal string
|
||||
description: Slot Index as decimal string
|
||||
|
||||
SlotAgent:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
$ref: "#/components/schemas/SlotId"
|
||||
slotIndex:
|
||||
type: string
|
||||
description: Slot Index as decimal string
|
||||
requestId:
|
||||
$ref: "#/components/schemas/Id"
|
||||
request:
|
||||
$ref: "#/components/schemas/StorageRequest"
|
||||
reservation:
|
||||
$ref: "#/components/schemas/Reservation"
|
||||
state:
|
||||
type: string
|
||||
description: Description of the slot's
|
||||
enum:
|
||||
- SaleCancelled
|
||||
- SaleDownloading
|
||||
- SaleErrored
|
||||
- SaleFailed
|
||||
- SaleFilled
|
||||
- SaleFilling
|
||||
- SaleFinished
|
||||
- SaleIgnored
|
||||
- SaleInitialProving
|
||||
- SalePayout
|
||||
- SalePreparing
|
||||
- SaleProving
|
||||
- SaleUnknown
|
||||
|
||||
Reservation:
|
||||
type: object
|
||||
@@ -183,21 +238,21 @@ components:
|
||||
$ref: "#/components/schemas/Id"
|
||||
slotIndex:
|
||||
type: string
|
||||
description: Slot Index as hexadecimal string
|
||||
description: Slot Index as decimal string
|
||||
|
||||
StorageRequestCreation:
|
||||
type: object
|
||||
required:
|
||||
- reward
|
||||
- pricePerBytePerSecond
|
||||
- duration
|
||||
- proofProbability
|
||||
- collateral
|
||||
- collateralPerByte
|
||||
- expiry
|
||||
properties:
|
||||
duration:
|
||||
$ref: "#/components/schemas/Duration"
|
||||
reward:
|
||||
$ref: "#/components/schemas/Reward"
|
||||
pricePerBytePerSecond:
|
||||
$ref: "#/components/schemas/PricePerBytePerSecond"
|
||||
proofProbability:
|
||||
$ref: "#/components/schemas/ProofProbability"
|
||||
nodes:
|
||||
@@ -208,16 +263,16 @@ components:
|
||||
description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
|
||||
type: integer
|
||||
default: 0
|
||||
collateral:
|
||||
collateralPerByte:
|
||||
type: string
|
||||
description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
|
||||
description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots
|
||||
expiry:
|
||||
type: string
|
||||
description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
|
||||
StorageAsk:
|
||||
type: object
|
||||
required:
|
||||
- reward
|
||||
- pricePerBytePerSecond
|
||||
properties:
|
||||
slots:
|
||||
description: Number of slots (eq. hosts) that the Request want to have the content spread over
|
||||
@@ -229,8 +284,8 @@ components:
|
||||
$ref: "#/components/schemas/Duration"
|
||||
proofProbability:
|
||||
$ref: "#/components/schemas/ProofProbability"
|
||||
reward:
|
||||
$ref: "#/components/schemas/Reward"
|
||||
pricePerBytePerSecond:
|
||||
$ref: "#/components/schemas/PricePerBytePerSecond"
|
||||
maxSlotLoss:
|
||||
type: integer
|
||||
description: Max slots that can be lost without data considered to be lost
|
||||
@@ -259,8 +314,18 @@ components:
|
||||
state:
|
||||
type: string
|
||||
description: Description of the Request's state
|
||||
enum:
|
||||
- cancelled
|
||||
- error
|
||||
- failed
|
||||
- finished
|
||||
- pending
|
||||
- started
|
||||
- submitted
|
||||
- unknown
|
||||
error:
|
||||
type: string
|
||||
nullable: true
|
||||
description: If Request failed, then here is presented the error message
|
||||
request:
|
||||
$ref: "#/components/schemas/StorageRequest"
|
||||
@@ -284,10 +349,10 @@ components:
|
||||
ManifestItem:
|
||||
type: object
|
||||
properties:
|
||||
rootHash:
|
||||
treeCid:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
description: "Root hash of the content"
|
||||
originalBytes:
|
||||
description: "Unique data identifier"
|
||||
datasetSize:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Length of original content in bytes"
|
||||
@@ -297,6 +362,16 @@ components:
|
||||
protected:
|
||||
type: boolean
|
||||
description: "Indicates if content is protected by erasure-coding"
|
||||
filename:
|
||||
type: string
|
||||
nullable: true
|
||||
description: "The original name of the uploaded content (optional)"
|
||||
example: codex.png
|
||||
mimetype:
|
||||
type: string
|
||||
nullable: true
|
||||
description: "The original mimetype of the uploaded content (optional)"
|
||||
example: image/png
|
||||
|
||||
Space:
|
||||
type: object
|
||||
@@ -308,15 +383,15 @@ components:
|
||||
quotaMaxBytes:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Maximum storage space used by the node"
|
||||
description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
|
||||
quotaUsedBytes:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Amount of storage space currently in use"
|
||||
description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
|
||||
quotaReservedBytes:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Amount of storage space reserved"
|
||||
description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
|
||||
|
||||
servers:
|
||||
- url: "http://localhost:8080/api/codex/v1"
|
||||
@@ -338,14 +413,14 @@ paths:
|
||||
description: |
|
||||
If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used
|
||||
to invoke peer discovery, if it succeeds the returned addresses will be used to dial.
|
||||
tags: [ Node ]
|
||||
tags: [Node]
|
||||
operationId: connectPeer
|
||||
parameters:
|
||||
- in: path
|
||||
name: peerId
|
||||
required: true
|
||||
schema:
|
||||
$ref: "#/components/schemas/PeerId"
|
||||
$ref: "#/components/schemas/PeerId"
|
||||
description: Peer that should be dialed.
|
||||
- in: query
|
||||
name: addrs
|
||||
@@ -368,7 +443,7 @@ paths:
|
||||
"/data":
|
||||
get:
|
||||
summary: "Lists manifest CIDs stored locally in node."
|
||||
tags: [ Data ]
|
||||
tags: [Data]
|
||||
operationId: listData
|
||||
responses:
|
||||
"200":
|
||||
@@ -382,12 +457,29 @@ paths:
|
||||
description: Invalid CID is specified
|
||||
"404":
|
||||
description: Content specified by the CID is not found
|
||||
"422":
|
||||
description: The content type is not a valid content type or the filename is not valid
|
||||
"500":
|
||||
description: Well it was bad-bad
|
||||
post:
|
||||
summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
|
||||
tags: [ Data ]
|
||||
tags: [Data]
|
||||
operationId: upload
|
||||
parameters:
|
||||
- name: content-type
|
||||
in: header
|
||||
required: false
|
||||
description: The content type of the file. Must be valid.
|
||||
schema:
|
||||
type: string
|
||||
example: "image/png"
|
||||
- name: content-disposition
|
||||
in: header
|
||||
required: false
|
||||
description: The content disposition used to send the filename.
|
||||
schema:
|
||||
type: string
|
||||
example: 'attachment; filename="codex.png"'
|
||||
requestBody:
|
||||
content:
|
||||
application/octet-stream:
|
||||
@@ -407,14 +499,14 @@ paths:
|
||||
"/data/{cid}":
|
||||
get:
|
||||
summary: "Download a file from the local node in a streaming manner. If the file is not available locally, a 404 is returned."
|
||||
tags: [ Data ]
|
||||
tags: [Data]
|
||||
operationId: downloadLocal
|
||||
parameters:
|
||||
- in: path
|
||||
name: cid
|
||||
required: true
|
||||
schema:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
$ref: "#/components/schemas/Cid"
|
||||
description: File to be downloaded.
|
||||
|
||||
responses:
|
||||
@@ -433,16 +525,42 @@ paths:
|
||||
description: Well it was bad-bad
|
||||
|
||||
"/data/{cid}/network":
|
||||
get:
|
||||
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
|
||||
tags: [ Data ]
|
||||
post:
|
||||
summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
|
||||
tags: [Data]
|
||||
operationId: downloadNetwork
|
||||
parameters:
|
||||
- in: path
|
||||
name: cid
|
||||
required: true
|
||||
schema:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
$ref: "#/components/schemas/Cid"
|
||||
description: "File to be downloaded."
|
||||
responses:
|
||||
"200":
|
||||
description: Manifest information for download that has been started.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DataItem"
|
||||
"400":
|
||||
description: Invalid CID is specified
|
||||
"404":
|
||||
description: Failed to download dataset manifest
|
||||
"500":
|
||||
description: Well it was bad-bad
|
||||
|
||||
"/data/{cid}/network/stream":
|
||||
get:
|
||||
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
|
||||
tags: [Data]
|
||||
operationId: downloadNetworkStream
|
||||
parameters:
|
||||
- in: path
|
||||
name: cid
|
||||
required: true
|
||||
schema:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
description: "File to be downloaded."
|
||||
responses:
|
||||
"200":
|
||||
@@ -459,10 +577,36 @@ paths:
|
||||
"500":
|
||||
description: Well it was bad-bad
|
||||
|
||||
"/data/{cid}/network/manifest":
|
||||
get:
|
||||
summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
|
||||
tags: [Data]
|
||||
operationId: downloadNetworkManifest
|
||||
parameters:
|
||||
- in: path
|
||||
name: cid
|
||||
required: true
|
||||
schema:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
description: "File for which the manifest is to be downloaded."
|
||||
responses:
|
||||
"200":
|
||||
description: Manifest information.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DataItem"
|
||||
"400":
|
||||
description: Invalid CID is specified
|
||||
"404":
|
||||
description: Failed to download dataset manifest
|
||||
"500":
|
||||
description: Well it was bad-bad
|
||||
|
||||
"/space":
|
||||
get:
|
||||
summary: "Gets a summary of the storage space allocation of the node."
|
||||
tags: [ Data ]
|
||||
tags: [Data]
|
||||
operationId: space
|
||||
responses:
|
||||
"200":
|
||||
@@ -478,7 +622,7 @@ paths:
|
||||
"/sales/slots":
|
||||
get:
|
||||
summary: "Returns active slots"
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
operationId: getActiveSlots
|
||||
responses:
|
||||
"200":
|
||||
@@ -491,12 +635,12 @@ paths:
|
||||
$ref: "#/components/schemas/Slot"
|
||||
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/sales/slots/{slotId}":
|
||||
get:
|
||||
summary: "Returns active slot with id {slotId} for the host"
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
operationId: getActiveSlotById
|
||||
parameters:
|
||||
- in: path
|
||||
@@ -511,7 +655,7 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Slot"
|
||||
$ref: "#/components/schemas/SlotAgent"
|
||||
|
||||
"400":
|
||||
description: Invalid or missing SlotId
|
||||
@@ -520,13 +664,13 @@ paths:
|
||||
description: Host is not in an active sale for the slot
|
||||
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/sales/availability":
|
||||
get:
|
||||
summary: "Returns storage that is for sale"
|
||||
tags: [ Marketplace ]
|
||||
operationId: getOfferedStorage
|
||||
tags: [Marketplace]
|
||||
operationId: getAvailabilities
|
||||
responses:
|
||||
"200":
|
||||
description: Retrieved storage availabilities of the node
|
||||
@@ -535,16 +679,16 @@ paths:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/SalesAvailability"
|
||||
$ref: "#/components/schemas/SalesAvailabilityREAD"
|
||||
"500":
|
||||
description: Error getting unused availabilities
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
post:
|
||||
summary: "Offers storage for sale"
|
||||
operationId: offerStorage
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
@@ -564,7 +708,7 @@ paths:
|
||||
"500":
|
||||
description: Error reserving availability
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
"/sales/availability/{id}":
|
||||
patch:
|
||||
summary: "Updates availability"
|
||||
@@ -572,7 +716,7 @@ paths:
|
||||
The new parameters will be only considered for new requests.
|
||||
Existing Requests linked to this Availability will continue as is.
|
||||
operationId: updateOfferedStorage
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
parameters:
|
||||
- in: path
|
||||
name: id
|
||||
@@ -597,14 +741,14 @@ paths:
|
||||
"500":
|
||||
description: Error reserving availability
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/sales/availability/{id}/reservations":
|
||||
patch:
|
||||
get:
|
||||
summary: "Get availability's reservations"
|
||||
description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
|
||||
operationId: getReservations
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
parameters:
|
||||
- in: path
|
||||
name: id
|
||||
@@ -628,12 +772,12 @@ paths:
|
||||
"500":
|
||||
description: Error getting reservations
|
||||
"503":
|
||||
description: Sales are unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/storage/request/{cid}":
|
||||
post:
|
||||
summary: "Creates a new Request for storage"
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
operationId: createStorageRequest
|
||||
parameters:
|
||||
- in: path
|
||||
@@ -659,12 +803,12 @@ paths:
|
||||
"404":
|
||||
description: Request ID not found
|
||||
"503":
|
||||
description: Purchasing is unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/storage/purchases":
|
||||
get:
|
||||
summary: "Returns list of purchase IDs"
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
operationId: getPurchases
|
||||
responses:
|
||||
"200":
|
||||
@@ -676,12 +820,12 @@ paths:
|
||||
items:
|
||||
type: string
|
||||
"503":
|
||||
description: Purchasing is unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/storage/purchases/{id}":
|
||||
get:
|
||||
summary: "Returns purchase details"
|
||||
tags: [ Marketplace ]
|
||||
tags: [Marketplace]
|
||||
operationId: getPurchase
|
||||
parameters:
|
||||
- in: path
|
||||
@@ -702,13 +846,13 @@ paths:
|
||||
"404":
|
||||
description: Purchase not found
|
||||
"503":
|
||||
description: Purchasing is unavailable
|
||||
description: Persistence is not enabled
|
||||
|
||||
"/node/spr":
|
||||
"/spr":
|
||||
get:
|
||||
summary: "Get Node's SPR"
|
||||
operationId: getSPR
|
||||
tags: [ Node ]
|
||||
tags: [Node]
|
||||
responses:
|
||||
"200":
|
||||
description: Node's SPR
|
||||
@@ -722,11 +866,11 @@ paths:
|
||||
"503":
|
||||
description: Node SPR not ready, try again later
|
||||
|
||||
"/node/peerid":
|
||||
"/peerid":
|
||||
get:
|
||||
summary: "Get Node's PeerID"
|
||||
operationId: getPeerId
|
||||
tags: [ Node ]
|
||||
tags: [Node]
|
||||
responses:
|
||||
"200":
|
||||
description: Node's Peer ID
|
||||
@@ -741,7 +885,7 @@ paths:
|
||||
"/debug/chronicles/loglevel":
|
||||
post:
|
||||
summary: "Set log level at run time"
|
||||
tags: [ Debug ]
|
||||
tags: [Debug]
|
||||
operationId: setDebugLogLevel
|
||||
|
||||
parameters:
|
||||
@@ -763,11 +907,11 @@ paths:
|
||||
get:
|
||||
summary: "Gets node information"
|
||||
operationId: getDebugInfo
|
||||
tags: [ Debug ]
|
||||
tags: [Debug]
|
||||
responses:
|
||||
"200":
|
||||
description: Node's information
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DebugInfo"
|
||||
$ref: "#/components/schemas/DebugInfo"
|
||||
@@ -1,4 +1,5 @@
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using System.Collections.Generic;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin.ChainMonitor
|
||||
@@ -12,7 +13,9 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
RequestCancelledEventDTO[] cancelled,
|
||||
RequestFailedEventDTO[] failed,
|
||||
SlotFilledEventDTO[] slotFilled,
|
||||
SlotFreedEventDTO[] slotFreed
|
||||
SlotFreedEventDTO[] slotFreed,
|
||||
SlotReservationsFullEventDTO[] slotReservationsFull,
|
||||
ProofSubmittedEventDTO[] proofSubmitted
|
||||
)
|
||||
{
|
||||
BlockInterval = blockInterval;
|
||||
@@ -22,6 +25,9 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
Failed = failed;
|
||||
SlotFilled = slotFilled;
|
||||
SlotFreed = slotFreed;
|
||||
SlotReservationsFull = slotReservationsFull;
|
||||
ProofSubmitted = proofSubmitted;
|
||||
All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull, ProofSubmitted);
|
||||
}
|
||||
|
||||
public BlockInterval BlockInterval { get; }
|
||||
@@ -31,21 +37,9 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
public RequestFailedEventDTO[] Failed { get; }
|
||||
public SlotFilledEventDTO[] SlotFilled { get; }
|
||||
public SlotFreedEventDTO[] SlotFreed { get; }
|
||||
|
||||
public IHasBlock[] All
|
||||
{
|
||||
get
|
||||
{
|
||||
var all = new List<IHasBlock>();
|
||||
all.AddRange(Requests);
|
||||
all.AddRange(Fulfilled);
|
||||
all.AddRange(Cancelled);
|
||||
all.AddRange(Failed);
|
||||
all.AddRange(SlotFilled);
|
||||
all.AddRange(SlotFreed);
|
||||
return all.ToArray();
|
||||
}
|
||||
}
|
||||
public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
|
||||
public ProofSubmittedEventDTO[] ProofSubmitted { get; }
|
||||
public IHasBlock[] All { get; }
|
||||
|
||||
public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
|
||||
{
|
||||
@@ -66,8 +60,20 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
events.GetRequestCancelledEvents(),
|
||||
events.GetRequestFailedEvents(),
|
||||
events.GetSlotFilledEvents(),
|
||||
events.GetSlotFreedEvents()
|
||||
events.GetSlotFreedEvents(),
|
||||
events.GetSlotReservationsFullEvents(),
|
||||
events.GetProofSubmittedEvents()
|
||||
);
|
||||
}
|
||||
|
||||
private T[] ConcatAll<T>(params T[][] arrays)
|
||||
{
|
||||
var result = Array.Empty<T>();
|
||||
foreach (var array in arrays)
|
||||
{
|
||||
result = result.Concat(array).ToArray();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using BlockchainUtils;
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using GethPlugin;
|
||||
using Logging;
|
||||
using NethereumWorkflow.BlockUtils;
|
||||
using System.Numerics;
|
||||
using Utils;
|
||||
|
||||
@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
void OnRequestFailed(RequestEvent requestEvent);
|
||||
void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
|
||||
void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
|
||||
void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
|
||||
void OnProofSubmitted(BlockTimeEntry block, string id);
|
||||
void OnError(string msg);
|
||||
}
|
||||
|
||||
public class RequestEvent
|
||||
@@ -36,42 +39,52 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
private readonly ILog log;
|
||||
private readonly ICodexContracts contracts;
|
||||
private readonly IChainStateChangeHandler handler;
|
||||
private readonly bool doProofPeriodMonitoring;
|
||||
|
||||
public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
|
||||
public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring)
|
||||
{
|
||||
this.log = new LogPrefixer(log, "(ChainState) ");
|
||||
this.contracts = contracts;
|
||||
handler = changeHandler;
|
||||
this.doProofPeriodMonitoring = doProofPeriodMonitoring;
|
||||
TotalSpan = new TimeRange(startUtc, startUtc);
|
||||
PeriodMonitor = new PeriodMonitor(this.log, contracts);
|
||||
}
|
||||
|
||||
public TimeRange TotalSpan { get; private set; }
|
||||
public IChainStateRequest[] Requests => requests.ToArray();
|
||||
public PeriodMonitor PeriodMonitor { get; }
|
||||
|
||||
public void Update()
|
||||
public int Update()
|
||||
{
|
||||
Update(DateTime.UtcNow);
|
||||
return Update(DateTime.UtcNow);
|
||||
}
|
||||
|
||||
public void Update(DateTime toUtc)
|
||||
public int Update(DateTime toUtc)
|
||||
{
|
||||
var span = new TimeRange(TotalSpan.To, toUtc);
|
||||
var events = ChainEvents.FromTimeRange(contracts, span);
|
||||
Apply(events);
|
||||
|
||||
TotalSpan = new TimeRange(TotalSpan.From, span.To);
|
||||
return events.All.Length;
|
||||
}
|
||||
|
||||
private void Apply(ChainEvents events)
|
||||
{
|
||||
if (events.BlockInterval.TimeRange.From < TotalSpan.From)
|
||||
throw new Exception("Attempt to update ChainState with set of events from before its current record.");
|
||||
{
|
||||
var msg = "Attempt to update ChainState with set of events from before its current record.";
|
||||
handler.OnError(msg);
|
||||
throw new Exception(msg);
|
||||
}
|
||||
|
||||
log.Log($"ChainState updating: {events.BlockInterval}");
|
||||
log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
|
||||
|
||||
// Run through each block and apply the events to the state in order.
|
||||
var span = events.BlockInterval.TimeRange.Duration;
|
||||
var numBlocks = events.BlockInterval.NumberOfBlocks;
|
||||
if (numBlocks == 0) return;
|
||||
var spanPerBlock = span / numBlocks;
|
||||
|
||||
var eventUtc = events.BlockInterval.TimeRange.From;
|
||||
@@ -79,11 +92,18 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
{
|
||||
var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
|
||||
ApplyEvents(b, blockEvents, eventUtc);
|
||||
UpdatePeriodMonitor(b, eventUtc);
|
||||
|
||||
eventUtc += spanPerBlock;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdatePeriodMonitor(ulong blockNumber, DateTime eventUtc)
|
||||
{
|
||||
if (!doProofPeriodMonitoring) return;
|
||||
PeriodMonitor.Update(blockNumber, eventUtc, Requests);
|
||||
}
|
||||
|
||||
private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
|
||||
{
|
||||
foreach (var e in blockEvents)
|
||||
@@ -108,7 +128,7 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
|
||||
private void ApplyEvent(RequestFulfilledEventDTO @event)
|
||||
{
|
||||
var r = FindRequest(@event.RequestId);
|
||||
var r = FindRequest(@event);
|
||||
if (r == null) return;
|
||||
r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
|
||||
handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
|
||||
@@ -116,7 +136,7 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
|
||||
private void ApplyEvent(RequestCancelledEventDTO @event)
|
||||
{
|
||||
var r = FindRequest(@event.RequestId);
|
||||
var r = FindRequest(@event);
|
||||
if (r == null) return;
|
||||
r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
|
||||
handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
|
||||
@@ -124,7 +144,7 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
|
||||
private void ApplyEvent(RequestFailedEventDTO @event)
|
||||
{
|
||||
var r = FindRequest(@event.RequestId);
|
||||
var r = FindRequest(@event);
|
||||
if (r == null) return;
|
||||
r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
|
||||
handler.OnRequestFailed(new RequestEvent(@event.Block, r));
|
||||
@@ -132,7 +152,7 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
|
||||
private void ApplyEvent(SlotFilledEventDTO @event)
|
||||
{
|
||||
var r = FindRequest(@event.RequestId);
|
||||
var r = FindRequest(@event);
|
||||
if (r == null) return;
|
||||
r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
|
||||
r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
|
||||
@@ -141,13 +161,28 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
|
||||
private void ApplyEvent(SlotFreedEventDTO @event)
|
||||
{
|
||||
var r = FindRequest(@event.RequestId);
|
||||
var r = FindRequest(@event);
|
||||
if (r == null) return;
|
||||
r.Hosts.RemoveHost((int)@event.SlotIndex);
|
||||
r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
|
||||
handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
|
||||
}
|
||||
|
||||
private void ApplyEvent(SlotReservationsFullEventDTO @event)
|
||||
{
|
||||
var r = FindRequest(@event);
|
||||
if (r == null) return;
|
||||
r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
|
||||
handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
|
||||
}
|
||||
|
||||
private void ApplyEvent(ProofSubmittedEventDTO @event)
|
||||
{
|
||||
var id = Base58.Encode(@event.Id);
|
||||
log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id})");
|
||||
handler.OnProofSubmitted(@event.Block, id);
|
||||
}
|
||||
|
||||
private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
|
||||
{
|
||||
foreach (var r in requests)
|
||||
@@ -161,10 +196,23 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
}
|
||||
}
|
||||
|
||||
private ChainStateRequest? FindRequest(byte[] requestId)
|
||||
private ChainStateRequest? FindRequest(IHasRequestId request)
|
||||
{
|
||||
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
|
||||
if (r == null) log.Log("Unable to find request by ID!");
|
||||
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
|
||||
if (r == null)
|
||||
{
|
||||
var blockNumber = "unknown";
|
||||
if (request is IHasBlock blk)
|
||||
{
|
||||
blockNumber = blk.Block.BlockNumber.ToString();
|
||||
}
|
||||
|
||||
var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
|
||||
$"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
|
||||
|
||||
log.Error(msg);
|
||||
handler.OnError(msg);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
using GethPlugin;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using BlockchainUtils;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin.ChainMonitor
|
||||
{
|
||||
@@ -51,5 +47,20 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
{
|
||||
foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
|
||||
}
|
||||
|
||||
public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
|
||||
{
|
||||
foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
|
||||
}
|
||||
|
||||
public void OnError(string msg)
|
||||
{
|
||||
foreach (var handler in Handlers) handler.OnError(msg);
|
||||
}
|
||||
|
||||
public void OnProofSubmitted(BlockTimeEntry block, string id)
|
||||
{
|
||||
foreach (var handler in Handlers) handler.OnProofSubmitted(block, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using GethPlugin;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin.ChainMonitor
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using GethPlugin;
|
||||
using BlockchainUtils;
|
||||
using System.Numerics;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin.ChainMonitor
|
||||
{
|
||||
@@ -32,5 +33,17 @@ namespace CodexContractsPlugin.ChainMonitor
|
||||
public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
|
||||
{
|
||||
}
|
||||
|
||||
public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
|
||||
{
|
||||
}
|
||||
|
||||
public void OnError(string msg)
|
||||
{
|
||||
}
|
||||
|
||||
public void OnProofSubmitted(BlockTimeEntry block, string id)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin.ChainMonitor
|
||||
{
|
||||
public class PeriodMonitor
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly ICodexContracts contracts;
|
||||
private readonly List<PeriodReport> reports = new List<PeriodReport>();
|
||||
private ulong? currentPeriod = null;
|
||||
|
||||
public PeriodMonitor(ILog log, ICodexContracts contracts)
|
||||
{
|
||||
this.log = log;
|
||||
this.contracts = contracts;
|
||||
}
|
||||
|
||||
public void Update(ulong blockNumber, DateTime eventUtc, IChainStateRequest[] requests)
|
||||
{
|
||||
var period = contracts.GetPeriodNumber(eventUtc);
|
||||
if (!currentPeriod.HasValue)
|
||||
{
|
||||
currentPeriod = period;
|
||||
return;
|
||||
}
|
||||
if (period == currentPeriod.Value) return;
|
||||
|
||||
CreateReportForPeriod(blockNumber - 1, currentPeriod.Value, requests);
|
||||
currentPeriod = period;
|
||||
}
|
||||
|
||||
public PeriodReport[] GetAndClearReports()
|
||||
{
|
||||
var result = reports.ToArray();
|
||||
reports.Clear();
|
||||
return result;
|
||||
}
|
||||
|
||||
private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests)
|
||||
{
|
||||
log.Log("Creating report for period " + periodNumber);
|
||||
|
||||
ulong total = 0;
|
||||
ulong required = 0;
|
||||
var missed = new List<PeriodProofMissed>();
|
||||
foreach (var request in requests)
|
||||
{
|
||||
for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++)
|
||||
{
|
||||
var state = contracts.GetProofState(request.Request, slotIndex, lastBlockInPeriod, periodNumber);
|
||||
|
||||
total++;
|
||||
if (state.Required)
|
||||
{
|
||||
required++;
|
||||
if (state.Missing)
|
||||
{
|
||||
var idx = Convert.ToInt32(slotIndex);
|
||||
var host = request.Hosts.GetHost(idx);
|
||||
missed.Add(new PeriodProofMissed(host, request, idx));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
reports.Add(new PeriodReport(periodNumber, total, required, missed.ToArray()));
|
||||
}
|
||||
}
|
||||
|
||||
public class PeriodReport
|
||||
{
|
||||
public PeriodReport(ulong periodNumber, ulong totalNumSlots, ulong totalProofsRequired, PeriodProofMissed[] missedProofs)
|
||||
{
|
||||
PeriodNumber = periodNumber;
|
||||
TotalNumSlots = totalNumSlots;
|
||||
TotalProofsRequired = totalProofsRequired;
|
||||
MissedProofs = missedProofs;
|
||||
}
|
||||
|
||||
public ulong PeriodNumber { get; }
|
||||
public ulong TotalNumSlots { get; }
|
||||
public ulong TotalProofsRequired { get; }
|
||||
public PeriodProofMissed[] MissedProofs { get; }
|
||||
}
|
||||
|
||||
public class PeriodProofMissed
|
||||
{
|
||||
public PeriodProofMissed(EthAddress? host, IChainStateRequest request, int slotIndex)
|
||||
{
|
||||
Host = host;
|
||||
Request = request;
|
||||
SlotIndex = slotIndex;
|
||||
}
|
||||
|
||||
public EthAddress? Host { get; }
|
||||
public IChainStateRequest Request { get; }
|
||||
public int SlotIndex { get; }
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,11 @@
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using BlockchainUtils;
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using GethPlugin;
|
||||
using Logging;
|
||||
using Nethereum.ABI;
|
||||
using Nethereum.ABI.FunctionEncoding.Attributes;
|
||||
using Nethereum.Contracts;
|
||||
using Nethereum.Hex.HexConvertors.Extensions;
|
||||
using Nethereum.Util;
|
||||
using NethereumWorkflow;
|
||||
using Newtonsoft.Json;
|
||||
@@ -24,6 +28,21 @@ namespace CodexContractsPlugin
|
||||
ICodexContractsEvents GetEvents(BlockInterval blockInterval);
|
||||
EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
|
||||
RequestState GetRequestState(Request request);
|
||||
ulong GetPeriodNumber(DateTime utc);
|
||||
void WaitUntilNextPeriod();
|
||||
ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period);
|
||||
}
|
||||
|
||||
public class ProofState
|
||||
{
|
||||
public ProofState(bool required, bool missing)
|
||||
{
|
||||
Required = required;
|
||||
Missing = missing;
|
||||
}
|
||||
|
||||
public bool Required { get; }
|
||||
public bool Missing { get; }
|
||||
}
|
||||
|
||||
[JsonConverter(typeof(StringEnumConverter))]
|
||||
@@ -86,19 +105,23 @@ namespace CodexContractsPlugin
|
||||
return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
|
||||
}
|
||||
|
||||
public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
|
||||
public byte[] GetSlotId(Request request, decimal slotIndex)
|
||||
{
|
||||
var encoder = new ABIEncode();
|
||||
var encoded = encoder.GetABIEncoded(
|
||||
new ABIValue("bytes32", storageRequest.RequestId),
|
||||
new ABIValue("bytes32", request.RequestId),
|
||||
new ABIValue("uint256", slotIndex.ToBig())
|
||||
);
|
||||
|
||||
var hashed = Sha3Keccack.Current.CalculateHash(encoded);
|
||||
return Sha3Keccack.Current.CalculateHash(encoded);
|
||||
}
|
||||
|
||||
public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
|
||||
{
|
||||
var slotId = GetSlotId(storageRequest, slotIndex);
|
||||
var func = new GetHostFunction
|
||||
{
|
||||
SlotId = hashed
|
||||
SlotId = slotId
|
||||
};
|
||||
var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
|
||||
if (string.IsNullOrEmpty(address)) return null;
|
||||
@@ -114,6 +137,70 @@ namespace CodexContractsPlugin
|
||||
return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
|
||||
}
|
||||
|
||||
public ulong GetPeriodNumber(DateTime utc)
|
||||
{
|
||||
DateTimeOffset utco = DateTime.SpecifyKind(utc, DateTimeKind.Utc);
|
||||
var now = utco.ToUnixTimeSeconds();
|
||||
var periodSeconds = (int)Deployment.Config.Proofs.Period;
|
||||
var result = now / periodSeconds;
|
||||
return Convert.ToUInt64(result);
|
||||
}
|
||||
|
||||
public void WaitUntilNextPeriod()
|
||||
{
|
||||
log.Log("Waiting until next proof period...");
|
||||
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
||||
var periodSeconds = (int)Deployment.Config.Proofs.Period;
|
||||
var secondsLeft = now % periodSeconds;
|
||||
Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
|
||||
}
|
||||
|
||||
public ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period)
|
||||
{
|
||||
var slotId = GetSlotId(storageRequest, slotIndex);
|
||||
|
||||
var required = IsProofRequired(slotId, blockNumber);
|
||||
if (!required) return new ProofState(false, false);
|
||||
|
||||
var missing = IsProofMissing(slotId, blockNumber, period);
|
||||
return new ProofState(required, missing);
|
||||
}
|
||||
|
||||
private bool IsProofRequired(byte[] slotId, ulong blockNumber)
|
||||
{
|
||||
var func = new IsProofRequiredFunction
|
||||
{
|
||||
Id = slotId
|
||||
};
|
||||
var result = gethNode.Call<IsProofRequiredFunction, IsProofRequiredOutputDTO>(Deployment.MarketplaceAddress, func, blockNumber);
|
||||
return result.ReturnValue1;
|
||||
}
|
||||
|
||||
private bool IsProofMissing(byte[] slotId, ulong blockNumber, ulong period)
|
||||
{
|
||||
try
|
||||
{
|
||||
var funcB = new MarkProofAsMissingFunction
|
||||
{
|
||||
SlotId = slotId,
|
||||
Period = period
|
||||
};
|
||||
gethNode.Call<MarkProofAsMissingFunction>(Deployment.MarketplaceAddress, funcB, blockNumber);
|
||||
}
|
||||
catch (AggregateException exc)
|
||||
{
|
||||
if (exc.InnerExceptions.Count == 1)
|
||||
{
|
||||
if (exc.InnerExceptions[0].GetType() == typeof(SmartContractCustomErrorRevertException))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
throw;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private ContractInteractions StartInteraction()
|
||||
{
|
||||
return new ContractInteractions(log, gethNode);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
using GethPlugin;
|
||||
using KubernetesWorkflow;
|
||||
using KubernetesWorkflow.Recipe;
|
||||
using Logging;
|
||||
|
||||
namespace CodexContractsPlugin
|
||||
{
|
||||
@@ -19,7 +18,7 @@ namespace CodexContractsPlugin
|
||||
{
|
||||
var config = startupConfig.Get<CodexContractsContainerConfig>();
|
||||
|
||||
var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
|
||||
var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
|
||||
|
||||
SetSchedulingAffinity(notIn: "false");
|
||||
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
namespace CodexContractsPlugin
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
|
||||
namespace CodexContractsPlugin
|
||||
{
|
||||
public class CodexContractsDeployment
|
||||
{
|
||||
public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
|
||||
public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
|
||||
{
|
||||
Config = config;
|
||||
MarketplaceAddress = marketplaceAddress;
|
||||
Abi = abi;
|
||||
TokenAddress = tokenAddress;
|
||||
}
|
||||
|
||||
public MarketplaceConfig Config { get; }
|
||||
public string MarketplaceAddress { get; }
|
||||
public string Abi { get; }
|
||||
public string TokenAddress { get; }
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using BlockchainUtils;
|
||||
using CodexContractsPlugin.Marketplace;
|
||||
using GethPlugin;
|
||||
using Logging;
|
||||
using Nethereum.Contracts;
|
||||
using Nethereum.Hex.HexTypes;
|
||||
using NethereumWorkflow.BlockUtils;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin
|
||||
@@ -16,6 +17,8 @@ namespace CodexContractsPlugin
|
||||
RequestFailedEventDTO[] GetRequestFailedEvents();
|
||||
SlotFilledEventDTO[] GetSlotFilledEvents();
|
||||
SlotFreedEventDTO[] GetSlotFreedEvents();
|
||||
SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents();
|
||||
ProofSubmittedEventDTO[] GetProofSubmittedEvents();
|
||||
}
|
||||
|
||||
public class CodexContractsEvents : ICodexContractsEvents
|
||||
@@ -38,49 +41,32 @@ namespace CodexContractsPlugin
|
||||
{
|
||||
var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
var i = new ContractInteractions(log, gethNode);
|
||||
return events
|
||||
.Select(e =>
|
||||
return events.Select(e =>
|
||||
{
|
||||
var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
|
||||
var result = requestEvent.ReturnValue1;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
result.RequestId = e.Event.RequestId;
|
||||
return result;
|
||||
})
|
||||
.ToArray();
|
||||
var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
|
||||
var result = requestEvent.ReturnValue1;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
result.RequestId = e.Event.RequestId;
|
||||
return result;
|
||||
}).ToArray();
|
||||
}
|
||||
|
||||
public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
|
||||
{
|
||||
var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
return events.Select(e =>
|
||||
{
|
||||
var result = e.Event;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
return result;
|
||||
}).ToArray();
|
||||
return events.Select(SetBlockOnEvent).ToArray();
|
||||
}
|
||||
|
||||
public RequestCancelledEventDTO[] GetRequestCancelledEvents()
|
||||
{
|
||||
var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
return events.Select(e =>
|
||||
{
|
||||
var result = e.Event;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
return result;
|
||||
}).ToArray();
|
||||
return events.Select(SetBlockOnEvent).ToArray();
|
||||
}
|
||||
|
||||
public RequestFailedEventDTO[] GetRequestFailedEvents()
|
||||
{
|
||||
var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
return events.Select(e =>
|
||||
{
|
||||
var result = e.Event;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
return result;
|
||||
}).ToArray();
|
||||
return events.Select(SetBlockOnEvent).ToArray();
|
||||
}
|
||||
|
||||
public SlotFilledEventDTO[] GetSlotFilledEvents()
|
||||
@@ -98,12 +84,26 @@ namespace CodexContractsPlugin
|
||||
public SlotFreedEventDTO[] GetSlotFreedEvents()
|
||||
{
|
||||
var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
return events.Select(e =>
|
||||
{
|
||||
var result = e.Event;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
return result;
|
||||
}).ToArray();
|
||||
return events.Select(SetBlockOnEvent).ToArray();
|
||||
}
|
||||
|
||||
public SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents()
|
||||
{
|
||||
var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
return events.Select(SetBlockOnEvent).ToArray();
|
||||
}
|
||||
|
||||
public ProofSubmittedEventDTO[] GetProofSubmittedEvents()
|
||||
{
|
||||
var events = gethNode.GetEvents<ProofSubmittedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
|
||||
return events.Select(SetBlockOnEvent).ToArray();
|
||||
}
|
||||
|
||||
private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
|
||||
{
|
||||
var result = e.Event;
|
||||
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
|
||||
return result;
|
||||
}
|
||||
|
||||
private BlockTimeEntry GetBlock(ulong number)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user