Multi-platform nodejs bindings (#242)

2023-03-28 11:01:07 -05:00 · 2023-03-28 11:01:07 -05:00 · c5920c4ef4
parent 3c6b9346b3
commit c5920c4ef4
9 changed files with 162 additions and 166 deletions
--- a/.github/workflows/nodejs-tests.yml
+++ b/.github/workflows/nodejs-tests.yml
@ -8,13 +8,28 @@ on:
      - main

 jobs:
-  tests:
-    runs-on: ubuntu-latest
+  test:
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        node:
+          - 16
+          - 18
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive
-      - name: Setup
-        run: cd src && make blst
-      - name: Test
-        run: cd bindings/node.js && make build test
+      - name: Setup Node.js ${{matrix.node}}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{matrix.node}}
+      - name: Build/test bindings
+        working-directory: bindings/node.js
+        run: make
+      - name: Install distribution
+        working-directory: bindings/node.js/dist
+        run: npm install
--- a/bindings/node.js/CONTRIBUTING.md
+++ b/bindings/node.js/CONTRIBUTING.md
@ -24,22 +24,22 @@ make # Build bindings and verify build worked

 ## Project Commands

-* `make clean` - cleans artifacts
-* `make build` - prepares assets and builds bindings
-* `make test` - runs unit tests
-* `make format` - lints code
-* `make bundle` - builds `dist` for publishing
-* `make publish` - runs `npm publish`
+- `make clean` - cleans artifacts
+- `make build` - prepares assets and builds bindings
+- `make test` - runs unit tests
+- `make format` - lints code
+- `make bundle` - builds `dist` for publishing
+- `make publish` - runs `npm publish`

 ## `n-api` and `node-addon-api`

 There are two different flavors of abi-stable node addons.
 [n-api](https://nodejs.org/api/n-api.html) is the `C` api that is natively
-exported by `node.js`.  There is also a header-only `C++` implementation of the
+exported by `node.js`. There is also a header-only `C++` implementation of the
 `n-api` called [node-addon-api](https://github.com/nodejs/node-addon-api).
 There is mixed usage of the two in this library.

 The addon was built to be
 [context-aware](https://nodejs.github.io/node-addon-examples/special-topics/context-awareness/)
-so it will be safe to run on a worker thread.  Be sure not to use any
+so it will be safe to run on a worker thread. Be sure not to use any
 static/global variables as those are not thread safe.
--- a/bindings/node.js/Dockerfile
+++ b/bindings/node.js/Dockerfile
@ -1,23 +1,12 @@
-# Exists as a test harness for building and running tests in Linux
-
-
 FROM node:16-alpine
 RUN apk update && apk add --no-cache g++ make python3

+WORKDIR /app/bindings/node.js

-COPY ./dist/ /app/dist/
-COPY test.ts /app
-COPY testing_trusted_setups.json /app
-COPY kzg.ts /app
-COPY kzg.cxx /app
-COPY package.json /app
-COPY tsconfig.json /app
-COPY babel.config.js /app
-COPY jest.config.js /app
-COPY binding.dist.gyp /app/binding.gyp
-
-WORKDIR /app
-
+COPY dist .
 RUN yarn install

+COPY test ./test
+COPY jest.config.js .
+COPY ref-tests ../../tests
 CMD ["yarn", "jest"]
--- a/bindings/node.js/Makefile
+++ b/bindings/node.js/Makefile
@ -1,40 +1,78 @@
-all: clean build format test bundle
+# For a less verbose yarn.
+YARN = yarn --silent

+.PHONY: all
+all: format build test bundle
+
+# Cleans native dependency, bindings and typescript artifacts
+.PHONY: clean
 clean:
-	rm -rf build
-	rm -rf dist
-	rm -f *.node
-	rm -f *.a
-	rm -f *.o
-	rm -rf node_modules
+	@rm -rf build
+	@rm -rf dist
+	@rm -rf deps
+	@rm -f *.node
+	@rm -f *.a
+	@rm -f *.o
+	@rm -rf ref-tests

-build: src/kzg.cxx lib/kzg.ts package.json binding.gyp Makefile
-	cd ../../src && make c_kzg_4844.o && cp c_kzg_4844.o ../bindings/node.js
-	yarn install
-	yarn node-gyp rebuild
+# Cleans typescript dependencies
+.PHONY: clean-install
+clean-install:
+	@rm -rf node_modules

-test: build
-	yarn jest
+# Installs typescript dependencies
+.PHONY: install
+install:
+	@$(YARN) install --ignore-scripts

-format:
-	yarn prettier --write .
+# Cleans and rebuilds native dependencies, bindings and typescript wrapper
+.PHONY: build
+build: install clean
+	@# Prepare the dependencies directory
+	@mkdir -p deps/c-kzg
+	@cp -r ../../blst deps
+	@cp ../../src/c_kzg_4844.c deps/c-kzg
+	@cp ../../src/c_kzg_4844.h deps/c-kzg
+	@# Patch the blst_aux.h to fix a compiler error
+	@awk '{gsub(/typedef struct \{\} blst_uniq;/, "typedef struct { int dummy; } blst_uniq;")}1' \
+		deps/blst/bindings/blst_aux.h > blst_aux_temp.h
+	@mv blst_aux_temp.h deps/blst/bindings/blst_aux.h
+	@# Build the bindings
+	@$(YARN) node-gyp --loglevel=warn configure
+	@$(YARN) node-gyp --loglevel=warn build
+	@$(YARN) tsc -p tsconfig.build.json

-bundle: clean
-	mkdir dist
-	cp README.md dist/README.md
-	cp package.json dist/package.json
-	cp binding.dist.gyp dist/binding.gyp
-	node_modules/.bin/tsc -p tsconfig.build.json
-	cp -r src dist/src
-	mkdir -p dist/deps/c-kzg
-	cp -r ../../blst dist/deps
-	cp ../../src/c_kzg_4844.c dist/deps/c-kzg
-	cp ../../src/c_kzg_4844.h dist/deps/c-kzg
+# Bundle the distribution, also helpful for cross compatibility checks
+.PHONY: bundle
+bundle: build
+	@mv deps dist
+	@cp -r src dist/src
+	@cp README.md dist/README.md
+	@cp package.json dist/package.json
+	@cp binding.gyp dist/binding.gyp

-publish: bundle
-	cd dist
-	npm publish
+# Run unit tests and ref-tests
+.PHONY: test
+test: install
+	@echo
+	@$(YARN) jest

-linux-test: bundle
-	docker build -t "linux-test" .
-	docker logs --follow `docker run -d linux-test`
+# Lint js/ts code
+.PHONY: format
+format: install
+	@$(YARN) prettier --loglevel=warn --write .
+
+# Publish package to npm (requires an auth token)
+.PHONY: publish
+publish: build
+	@cd dist
+	@npm publish
+
+# Run ref-tests in linux environment for cross-compatability check
+.PHONY: linux-test
+linux-test: build
+	# Docker cannot copy from outside this dir
+	@cp -r ../../tests ref-tests
+	@docker build -t "linux-test" .
+	@docker logs --follow `docker run -d linux-test`
+	@rm -rf ref-tests
--- a/bindings/node.js/binding.dist.gyp
+++ b/bindings/node.js/binding.dist.gyp
@ -1,52 +0,0 @@
-{
-  "targets": [
-    {
-      "target_name": "kzg",
-      "cflags!": ["-fno-exceptions"],
-      "cflags_cc!": ["-fno-exceptions"],
-      "xcode_settings": {
-        "CLANG_CXX_LIBRARY": "libc++",
-        "MACOSX_DEPLOYMENT_TARGET": "13.0"
-      },
-      "defines": [
-        "NAPI_DISABLE_CPP_EXCEPTIONS",
-        "FIELD_ELEMENTS_PER_BLOB=<!(echo ${FIELD_ELEMENTS_PER_BLOB:-4096})"
-      ],
-      "sources": ["src/kzg.cxx"],
-      "include_dirs": [
-        "<(module_root_dir)/deps/blst/bindings",
-        "<(module_root_dir)/deps/c-kzg",
-        "<!@(node -p \"require('node-addon-api').include\")"
-      ],
-      "libraries": [
-        "<(module_root_dir)/c_kzg_4844.o",
-        "<(module_root_dir)/libblst.a"
-      ],
-      "dependencies": ["<!(node -p \"require('node-addon-api').gyp\")"],
-      "actions": [
-        {
-          "action_name": "build_blst",
-          "inputs": ["<(module_root_dir)/deps/blst/build.sh"],
-          "outputs": ["<(module_root_dir)/libblst.a"],
-          "action": ["<(module_root_dir)/deps/blst/build.sh"]
-        },
-        {
-          "action_name": "build_ckzg",
-          "inputs": [
-            "<(module_root_dir)/deps/c-kzg/c_kzg_4844.c",
-            "<(module_root_dir)/libblst.a"
-          ],
-          "outputs": ["<(module_root_dir)/c_kzg_4844.o"],
-          "action": [
-            "cc",
-            "-I<(module_root_dir)/deps/blst/bindings",
-            "-DFIELD_ELEMENTS_PER_BLOB=<!(echo ${FIELD_ELEMENTS_PER_BLOB:-4096})",
-            "-O2",
-            "-c",
-            "<(module_root_dir)/deps/c-kzg/c_kzg_4844.c"
-          ]
-        }
-      ]
-    }
-  ]
-}
--- a/bindings/node.js/binding.gyp
+++ b/bindings/node.js/binding.gyp
@ -2,27 +2,45 @@
  "targets": [
    {
      "target_name": "kzg",
-      "cflags!": ["-fno-exceptions"],
-      "cflags_cc!": ["-fno-exceptions"],
-      "xcode_settings": {
-        "CLANG_CXX_LIBRARY": "libc++",
-        "MACOSX_DEPLOYMENT_TARGET": "13.0"
-      },
-      "defines": [
-        "NAPI_DISABLE_CPP_EXCEPTIONS",
-        "FIELD_ELEMENTS_PER_BLOB=<!(echo ${FIELD_ELEMENTS_PER_BLOB:-4096})"
+      "sources": [
+        "src/kzg.cxx",
+        "deps/blst/src/server.c",
+        "deps/c-kzg/c_kzg_4844.c"
      ],
-      "sources": ["src/kzg.cxx"],
      "include_dirs": [
-        "../../inc",
-        "../../src",
+        "<(module_root_dir)/deps/blst/bindings",
+        "<(module_root_dir)/deps/c-kzg",
        "<!@(node -p \"require('node-addon-api').include\")"
      ],
-      "libraries": [
-        "<(module_root_dir)/c_kzg_4844.o",
-        "<(module_root_dir)/../../lib/libblst.a"
-      ],
-      "dependencies": ["<!(node -p \"require('node-addon-api').gyp\")"]
+      "defines": ["NAPI_DISABLE_CPP_EXCEPTIONS"],
+      "conditions": [
+        ["OS!='win'", {
+          "sources": ["deps/blst/build/assembly.S"],
+          "defines": ["FIELD_ELEMENTS_PER_BLOB=<!(echo ${FIELD_ELEMENTS_PER_BLOB:-4096})"],
+          "cflags_cc": [
+            "-std=c++17",
+            "-fPIC"
+          ]
+        }],
+        ["OS=='win'", {
+          "sources": ["deps/blst/build/win64/*-x86_64.asm"],
+          "defines": [
+            "_CRT_SECURE_NO_WARNINGS",
+            "FIELD_ELEMENTS_PER_BLOB=<!(powershell -Command \"if ($env:FIELD_ELEMENTS_PER_BLOB) { $env:FIELD_ELEMENTS_PER_BLOB } else { 4096 }\")"
+          ],
+          "msbuild_settings": {
+            "ClCompile": {
+              "AdditionalOptions": ["/std:c++17"]
+            }
+          }
+        }],
+        ["OS=='mac'", {
+          "xcode_settings": {
+            "CLANG_CXX_LIBRARY": "libc++",
+            "MACOSX_DEPLOYMENT_TARGET": "13.0"
+          }
+        }]
+      ]
    }
  ]
 }
--- a/bindings/node.js/test/kzg.test.ts
+++ b/bindings/node.js/test/kzg.test.ts
@ -36,46 +36,35 @@ const SETUP_FILE_PATH = resolve(

 const MAX_TOP_BYTE = 114;

-const TEST_DIR = "../../tests";
+const BLOB_TO_KZG_COMMITMENT_TESTS =
+  "../../tests/blob_to_kzg_commitment/*/*/data.yaml";
+const COMPUTE_KZG_PROOF_TESTS = "../../tests/compute_kzg_proof/*/*/data.yaml";
+const COMPUTE_BLOB_KZG_PROOF_TESTS =
+  "../../tests/compute_blob_kzg_proof/*/*/data.yaml";
+const VERIFY_KZG_PROOF_TESTS = "../../tests/verify_kzg_proof/*/*/data.yaml";
+const VERIFY_BLOB_KZG_PROOF_TESTS =
+  "../../tests/verify_blob_kzg_proof/*/*/data.yaml";
+const VERIFY_BLOB_KZG_PROOF_BATCH_TESTS =
+  "../../tests/verify_blob_kzg_proof_batch/*/*/data.yaml";
+
 type BlobToKzgCommitmentTest = TestMeta<{ blob: string }, string>;
-const BLOB_TO_KZG_COMMITMENT_TESTS = join(
-  TEST_DIR,
-  "blob_to_kzg_commitment/*/*/data.yaml",
-);
 type ComputeKzgProofTest = TestMeta<{ blob: string; z: string }, string[]>;
-const COMPUTE_KZG_PROOF_TESTS = join(
-  TEST_DIR,
-  "compute_kzg_proof/*/*/data.yaml",
-);
 type ComputeBlobKzgProofTest = TestMeta<
  { blob: string; commitment: string },
  string
 >;
-const COMPUTE_BLOB_KZG_PROOF_TESTS = join(
-  TEST_DIR,
-  "compute_blob_kzg_proof/*/*/data.yaml",
-);
 type VerifyKzgProofTest = TestMeta<
  { commitment: string; y: string; z: string; proof: string },
  boolean
 >;
-const VERIFY_KZG_PROOF_TESTS = join(TEST_DIR, "verify_kzg_proof/*/*/data.yaml");
 type VerifyBlobKzgProofTest = TestMeta<
  { blob: string; commitment: string; proof: string },
  boolean
 >;
-const VERIFY_BLOB_KZG_PROOF_TESTS = join(
-  TEST_DIR,
-  "verify_blob_kzg_proof/*/*/data.yaml",
-);
 type VerifyBatchKzgProofTest = TestMeta<
  { blobs: string[]; commitments: string[]; proofs: string[] },
  boolean
 >;
-const VERIFY_BLOB_KZG_PROOF_BATCH_TESTS = join(
-  TEST_DIR,
-  "verify_blob_kzg_proof_batch/*/*/data.yaml",
-);

 const generateRandomBlob = () => {
  return new Uint8Array(
--- a/src/c_kzg_4844.c
+++ b/src/c_kzg_4844.c
@ -64,16 +64,16 @@ static const char *FIAT_SHAMIR_PROTOCOL_DOMAIN = "FSBLOBVERIFY_V1_";
 static const char *RANDOM_CHALLENGE_KZG_BATCH_DOMAIN = "RCKZGBATCH___V1_";

 /** Length of the domain strings above. */
-static const size_t DOMAIN_STR_LENGTH = 16;
+#define DOMAIN_STR_LENGTH 16

 /** The number of bytes in a g1 point. */
-static const size_t BYTES_PER_G1 = 48;
+#define BYTES_PER_G1 48

 /** The number of bytes in a g2 point. */
-static const size_t BYTES_PER_G2 = 96;
+#define BYTES_PER_G2 96

 /** The number of g2 points in a trusted setup. */
-static const size_t TRUSTED_SETUP_NUM_G2_POINTS = 65;
+#define TRUSTED_SETUP_NUM_G2_POINTS 65

 // clang-format off

@ -734,13 +734,12 @@ static C_KZG_RET blob_to_polynomial(Polynomial *p, const Blob *blob) {
 }

 /* Input size to the Fiat-Shamir challenge computation. */
-static const size_t CHALLENGE_INPUT_SIZE = DOMAIN_STR_LENGTH +
-                                           sizeof(uint64_t) + sizeof(uint64_t) +
-                                           BYTES_PER_BLOB +
-                                           BYTES_PER_COMMITMENT;
+#define CHALLENGE_INPUT_SIZE \
+    (DOMAIN_STR_LENGTH + 16 + BYTES_PER_BLOB + BYTES_PER_COMMITMENT)

 /**
- * Return the Fiat-Shamir challenge required to verify `blob` and `commitment`.
+ * Return the Fiat-Shamir challenge required to verify `blob` and
+ * `commitment`.
 *
 * @remark This function should compute challenges even if `n==0`.
 *
--- a/src/c_kzg_4844.h
+++ b/src/c_kzg_4844.h
@ -37,16 +37,16 @@ extern "C" {
 ///////////////////////////////////////////////////////////////////////////////

 #ifndef FIELD_ELEMENTS_PER_BLOB
-#error FIELD_ELEMENTS_PER_BLOB is undefined. This value must be externally supplied.
+#error FIELD_ELEMENTS_PER_BLOB must be defined
 #endif // FIELD_ELEMENTS_PER_BLOB
 /**
 * There are only 1<<32 2-adic roots of unity in the field, limiting the
 * possible values of FIELD_ELEMENTS_PER_BLOB. The restriction to 1<<31 is a
 * current implementation limitation. Notably, the size of the FFT setup would
- * overflow uint32_t, which would casues issues.
+ * overflow uint32_t, which would cause issues.
 */
-#if ((FIELD_ELEMENTS_PER_BLOB) <= 0) || ((FIELD_ELEMENTS_PER_BLOB) > (1 << 31))
-#error Invalid value of FIELD_ELEMENTS_PER_BLOB
+#if (FIELD_ELEMENTS_PER_BLOB <= 0) || (FIELD_ELEMENTS_PER_BLOB > (1UL << 31))
+#error FIELD_ELEMENTS_PER_BLOB must be between 1 and 2^31
 #endif // FIELD_ELEMENTS_PER_BLOB

 /**