From c5920c4ef432a85f8ddfd9a020ffdb0c660944b6 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Tue, 28 Mar 2023 11:01:07 -0500 Subject: [PATCH] Multi-platform nodejs bindings (#242) --- .github/workflows/nodejs-tests.yml | 27 ++++++-- bindings/node.js/CONTRIBUTING.md | 16 ++--- bindings/node.js/Dockerfile | 21 ++---- bindings/node.js/Makefile | 102 ++++++++++++++++++++--------- bindings/node.js/binding.dist.gyp | 52 --------------- bindings/node.js/binding.gyp | 52 ++++++++++----- bindings/node.js/test/kzg.test.ts | 33 ++++------ src/c_kzg_4844.c | 17 +++-- src/c_kzg_4844.h | 8 +-- 9 files changed, 162 insertions(+), 166 deletions(-) delete mode 100644 bindings/node.js/binding.dist.gyp diff --git a/.github/workflows/nodejs-tests.yml b/.github/workflows/nodejs-tests.yml index e446b95..5b73dde 100644 --- a/.github/workflows/nodejs-tests.yml +++ b/.github/workflows/nodejs-tests.yml @@ -8,13 +8,28 @@ on: - main jobs: - tests: - runs-on: ubuntu-latest + test: + runs-on: ${{matrix.os}} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + node: + - 16 + - 18 steps: - uses: actions/checkout@v3 with: submodules: recursive - - name: Setup - run: cd src && make blst - - name: Test - run: cd bindings/node.js && make build test + - name: Setup Node.js ${{matrix.node}} + uses: actions/setup-node@v1 + with: + node-version: ${{matrix.node}} + - name: Build/test bindings + working-directory: bindings/node.js + run: make + - name: Install distribution + working-directory: bindings/node.js/dist + run: npm install diff --git a/bindings/node.js/CONTRIBUTING.md b/bindings/node.js/CONTRIBUTING.md index dacf857..6168eda 100644 --- a/bindings/node.js/CONTRIBUTING.md +++ b/bindings/node.js/CONTRIBUTING.md @@ -24,22 +24,22 @@ make # Build bindings and verify build worked ## Project Commands -* `make clean` - cleans artifacts -* `make build` - prepares assets and builds bindings -* `make test` - runs unit tests -* `make format` - lints code -* `make bundle` - builds `dist` for publishing -* `make publish` - runs `npm publish` +- `make clean` - cleans artifacts +- `make build` - prepares assets and builds bindings +- `make test` - runs unit tests +- `make format` - lints code +- `make bundle` - builds `dist` for publishing +- `make publish` - runs `npm publish` ## `n-api` and `node-addon-api` There are two different flavors of abi-stable node addons. [n-api](https://nodejs.org/api/n-api.html) is the `C` api that is natively -exported by `node.js`. There is also a header-only `C++` implementation of the +exported by `node.js`. There is also a header-only `C++` implementation of the `n-api` called [node-addon-api](https://github.com/nodejs/node-addon-api). There is mixed usage of the two in this library. The addon was built to be [context-aware](https://nodejs.github.io/node-addon-examples/special-topics/context-awareness/) -so it will be safe to run on a worker thread. Be sure not to use any +so it will be safe to run on a worker thread. Be sure not to use any static/global variables as those are not thread safe. diff --git a/bindings/node.js/Dockerfile b/bindings/node.js/Dockerfile index 007cd4e..e5d2b89 100644 --- a/bindings/node.js/Dockerfile +++ b/bindings/node.js/Dockerfile @@ -1,23 +1,12 @@ -# Exists as a test harness for building and running tests in Linux - - FROM node:16-alpine RUN apk update && apk add --no-cache g++ make python3 +WORKDIR /app/bindings/node.js -COPY ./dist/ /app/dist/ -COPY test.ts /app -COPY testing_trusted_setups.json /app -COPY kzg.ts /app -COPY kzg.cxx /app -COPY package.json /app -COPY tsconfig.json /app -COPY babel.config.js /app -COPY jest.config.js /app -COPY binding.dist.gyp /app/binding.gyp - -WORKDIR /app - +COPY dist . RUN yarn install +COPY test ./test +COPY jest.config.js . +COPY ref-tests ../../tests CMD ["yarn", "jest"] diff --git a/bindings/node.js/Makefile b/bindings/node.js/Makefile index 0071df0..d244280 100644 --- a/bindings/node.js/Makefile +++ b/bindings/node.js/Makefile @@ -1,40 +1,78 @@ -all: clean build format test bundle +# For a less verbose yarn. +YARN = yarn --silent +.PHONY: all +all: format build test bundle + +# Cleans native dependency, bindings and typescript artifacts +.PHONY: clean clean: - rm -rf build - rm -rf dist - rm -f *.node - rm -f *.a - rm -f *.o - rm -rf node_modules + @rm -rf build + @rm -rf dist + @rm -rf deps + @rm -f *.node + @rm -f *.a + @rm -f *.o + @rm -rf ref-tests -build: src/kzg.cxx lib/kzg.ts package.json binding.gyp Makefile - cd ../../src && make c_kzg_4844.o && cp c_kzg_4844.o ../bindings/node.js - yarn install - yarn node-gyp rebuild +# Cleans typescript dependencies +.PHONY: clean-install +clean-install: + @rm -rf node_modules -test: build - yarn jest +# Installs typescript dependencies +.PHONY: install +install: + @$(YARN) install --ignore-scripts -format: - yarn prettier --write . +# Cleans and rebuilds native dependencies, bindings and typescript wrapper +.PHONY: build +build: install clean + @# Prepare the dependencies directory + @mkdir -p deps/c-kzg + @cp -r ../../blst deps + @cp ../../src/c_kzg_4844.c deps/c-kzg + @cp ../../src/c_kzg_4844.h deps/c-kzg + @# Patch the blst_aux.h to fix a compiler error + @awk '{gsub(/typedef struct \{\} blst_uniq;/, "typedef struct { int dummy; } blst_uniq;")}1' \ + deps/blst/bindings/blst_aux.h > blst_aux_temp.h + @mv blst_aux_temp.h deps/blst/bindings/blst_aux.h + @# Build the bindings + @$(YARN) node-gyp --loglevel=warn configure + @$(YARN) node-gyp --loglevel=warn build + @$(YARN) tsc -p tsconfig.build.json -bundle: clean - mkdir dist - cp README.md dist/README.md - cp package.json dist/package.json - cp binding.dist.gyp dist/binding.gyp - node_modules/.bin/tsc -p tsconfig.build.json - cp -r src dist/src - mkdir -p dist/deps/c-kzg - cp -r ../../blst dist/deps - cp ../../src/c_kzg_4844.c dist/deps/c-kzg - cp ../../src/c_kzg_4844.h dist/deps/c-kzg +# Bundle the distribution, also helpful for cross compatibility checks +.PHONY: bundle +bundle: build + @mv deps dist + @cp -r src dist/src + @cp README.md dist/README.md + @cp package.json dist/package.json + @cp binding.gyp dist/binding.gyp -publish: bundle - cd dist - npm publish +# Run unit tests and ref-tests +.PHONY: test +test: install + @echo + @$(YARN) jest -linux-test: bundle - docker build -t "linux-test" . - docker logs --follow `docker run -d linux-test` +# Lint js/ts code +.PHONY: format +format: install + @$(YARN) prettier --loglevel=warn --write . + +# Publish package to npm (requires an auth token) +.PHONY: publish +publish: build + @cd dist + @npm publish + +# Run ref-tests in linux environment for cross-compatability check +.PHONY: linux-test +linux-test: build + # Docker cannot copy from outside this dir + @cp -r ../../tests ref-tests + @docker build -t "linux-test" . + @docker logs --follow `docker run -d linux-test` + @rm -rf ref-tests diff --git a/bindings/node.js/binding.dist.gyp b/bindings/node.js/binding.dist.gyp deleted file mode 100644 index c9f1108..0000000 --- a/bindings/node.js/binding.dist.gyp +++ /dev/null @@ -1,52 +0,0 @@ -{ - "targets": [ - { - "target_name": "kzg", - "cflags!": ["-fno-exceptions"], - "cflags_cc!": ["-fno-exceptions"], - "xcode_settings": { - "CLANG_CXX_LIBRARY": "libc++", - "MACOSX_DEPLOYMENT_TARGET": "13.0" - }, - "defines": [ - "NAPI_DISABLE_CPP_EXCEPTIONS", - "FIELD_ELEMENTS_PER_BLOB=; -const BLOB_TO_KZG_COMMITMENT_TESTS = join( - TEST_DIR, - "blob_to_kzg_commitment/*/*/data.yaml", -); type ComputeKzgProofTest = TestMeta<{ blob: string; z: string }, string[]>; -const COMPUTE_KZG_PROOF_TESTS = join( - TEST_DIR, - "compute_kzg_proof/*/*/data.yaml", -); type ComputeBlobKzgProofTest = TestMeta< { blob: string; commitment: string }, string >; -const COMPUTE_BLOB_KZG_PROOF_TESTS = join( - TEST_DIR, - "compute_blob_kzg_proof/*/*/data.yaml", -); type VerifyKzgProofTest = TestMeta< { commitment: string; y: string; z: string; proof: string }, boolean >; -const VERIFY_KZG_PROOF_TESTS = join(TEST_DIR, "verify_kzg_proof/*/*/data.yaml"); type VerifyBlobKzgProofTest = TestMeta< { blob: string; commitment: string; proof: string }, boolean >; -const VERIFY_BLOB_KZG_PROOF_TESTS = join( - TEST_DIR, - "verify_blob_kzg_proof/*/*/data.yaml", -); type VerifyBatchKzgProofTest = TestMeta< { blobs: string[]; commitments: string[]; proofs: string[] }, boolean >; -const VERIFY_BLOB_KZG_PROOF_BATCH_TESTS = join( - TEST_DIR, - "verify_blob_kzg_proof_batch/*/*/data.yaml", -); const generateRandomBlob = () => { return new Uint8Array( diff --git a/src/c_kzg_4844.c b/src/c_kzg_4844.c index eb08152..6c9b6d7 100644 --- a/src/c_kzg_4844.c +++ b/src/c_kzg_4844.c @@ -64,16 +64,16 @@ static const char *FIAT_SHAMIR_PROTOCOL_DOMAIN = "FSBLOBVERIFY_V1_"; static const char *RANDOM_CHALLENGE_KZG_BATCH_DOMAIN = "RCKZGBATCH___V1_"; /** Length of the domain strings above. */ -static const size_t DOMAIN_STR_LENGTH = 16; +#define DOMAIN_STR_LENGTH 16 /** The number of bytes in a g1 point. */ -static const size_t BYTES_PER_G1 = 48; +#define BYTES_PER_G1 48 /** The number of bytes in a g2 point. */ -static const size_t BYTES_PER_G2 = 96; +#define BYTES_PER_G2 96 /** The number of g2 points in a trusted setup. */ -static const size_t TRUSTED_SETUP_NUM_G2_POINTS = 65; +#define TRUSTED_SETUP_NUM_G2_POINTS 65 // clang-format off @@ -734,13 +734,12 @@ static C_KZG_RET blob_to_polynomial(Polynomial *p, const Blob *blob) { } /* Input size to the Fiat-Shamir challenge computation. */ -static const size_t CHALLENGE_INPUT_SIZE = DOMAIN_STR_LENGTH + - sizeof(uint64_t) + sizeof(uint64_t) + - BYTES_PER_BLOB + - BYTES_PER_COMMITMENT; +#define CHALLENGE_INPUT_SIZE \ + (DOMAIN_STR_LENGTH + 16 + BYTES_PER_BLOB + BYTES_PER_COMMITMENT) /** - * Return the Fiat-Shamir challenge required to verify `blob` and `commitment`. + * Return the Fiat-Shamir challenge required to verify `blob` and + * `commitment`. * * @remark This function should compute challenges even if `n==0`. * diff --git a/src/c_kzg_4844.h b/src/c_kzg_4844.h index 77a1850..2862c93 100644 --- a/src/c_kzg_4844.h +++ b/src/c_kzg_4844.h @@ -37,16 +37,16 @@ extern "C" { /////////////////////////////////////////////////////////////////////////////// #ifndef FIELD_ELEMENTS_PER_BLOB -#error FIELD_ELEMENTS_PER_BLOB is undefined. This value must be externally supplied. +#error FIELD_ELEMENTS_PER_BLOB must be defined #endif // FIELD_ELEMENTS_PER_BLOB /** * There are only 1<<32 2-adic roots of unity in the field, limiting the * possible values of FIELD_ELEMENTS_PER_BLOB. The restriction to 1<<31 is a * current implementation limitation. Notably, the size of the FFT setup would - * overflow uint32_t, which would casues issues. + * overflow uint32_t, which would cause issues. */ -#if ((FIELD_ELEMENTS_PER_BLOB) <= 0) || ((FIELD_ELEMENTS_PER_BLOB) > (1 << 31)) -#error Invalid value of FIELD_ELEMENTS_PER_BLOB +#if (FIELD_ELEMENTS_PER_BLOB <= 0) || (FIELD_ELEMENTS_PER_BLOB > (1UL << 31)) +#error FIELD_ELEMENTS_PER_BLOB must be between 1 and 2^31 #endif // FIELD_ELEMENTS_PER_BLOB /**