Merge branch '4844_3038' into dgcoffman/nodejs-bindings

.gitignore

@@ -1,15 +1,6 @@
 *.o
 *.a
-*_test
-*_bench
-*_debug
-*_tune
-*.prof
 *.out
-*.log
-src/*_tune.c
-tmp/
-doc/
 inc/blst.h*
 inc/blst_aux.h*
 .vscode/

README.md

@@ -8,7 +8,7 @@ This is a copy of C-KZG stripped down to support the [Polynomial Commitments](ht
 
 We also provide `load_trusted_setup` and `free_trusted_setup` to load the
 trusted setup data from a file into an object that can be passed to the API
-functions, and conversions to/from byte arrays for the relevant types.
+functions, and functions for converting commitments/proofs/points to/from bytes.
 
 The only dependency is [blst](https://github.com/supranational/blst).
 Ensure `blst.h` is provided in `inc` and `libblst.a` in `lib`.

bindings/csharp/ckzg.c

@@ -22,59 +22,39 @@ void free_trusted_setup_wrap(KZGSettings *s) {
   free(s);
 }
 
-void blob_to_kzg_commitment_wrap(uint8_t out[48], const uint8_t blob[FIELD_ELEMENTS_PER_BLOB * 32], const KZGSettings *s) {
+void blob_to_kzg_commitment_wrap(uint8_t out[48], const Blob blob, const KZGSettings *s) {
-  Polynomial p;
-  for (size_t i = 0; i < FIELD_ELEMENTS_PER_BLOB; i++)
-    bytes_to_bls_field(&p[i], &blob[i * 32]);
-
   KZGCommitment c;
-  blob_to_kzg_commitment(&c, p, s);
+  blob_to_kzg_commitment(&c, blob, s);
-
   bytes_from_g1(out, &c);
 }
 
-int verify_aggregate_kzg_proof_wrap(const uint8_t blobs[], const uint8_t commitments[], size_t n, const uint8_t proof[48], const KZGSettings *s) {
+int verify_aggregate_kzg_proof_wrap(const Blob blobs[], const uint8_t commitments[], size_t n, const uint8_t proof[48], const KZGSettings *s) {
-  Polynomial* p = calloc(n, sizeof(Polynomial));
+  KZGProof f;
-  if (p == NULL) return -1;
+  C_KZG_RET ret;
+  ret = bytes_to_g1(&f, proof);
+  if (ret != C_KZG_OK) return -1;
 
   KZGCommitment* c = calloc(n, sizeof(KZGCommitment));
-  if (c == NULL) { free(p); return -1; }
+  if (c == NULL) return -2;
-
-  C_KZG_RET ret;
 
   for (size_t i = 0; i < n; i++) {
-    for (size_t j = 0; j < FIELD_ELEMENTS_PER_BLOB; j++)
-      bytes_to_bls_field(&p[i][j], &blobs[i * FIELD_ELEMENTS_PER_BLOB * 32 + j * 32]);
     ret = bytes_to_g1(&c[i], &commitments[i * 48]);
-    if (ret != C_KZG_OK) { free(c); free(p); return -1; }
+    if (ret != C_KZG_OK) { free(c); return -1; }
   }
 
-  KZGProof f;
-  ret = bytes_to_g1(&f, proof);
-  if (ret != C_KZG_OK) { free(c); free(p); return -1; }
-
   bool b;
-  ret = verify_aggregate_kzg_proof(&b, p, c, n, &f, s);
+  ret = verify_aggregate_kzg_proof(&b, blobs, c, n, &f, s);
-  if (ret != C_KZG_OK) { free(c); free(p); return -1; }
+  free(c);
+  if (ret != C_KZG_OK) return -1;
 
-  free(c); free(p);
   return b ? 0 : 1;
 }
 
-C_KZG_RET compute_aggregate_kzg_proof_wrap(uint8_t out[48], const uint8_t blobs[], size_t n, const KZGSettings *s) {
+C_KZG_RET compute_aggregate_kzg_proof_wrap(uint8_t out[48], const Blob blobs[], size_t n, const KZGSettings *s) {
-  Polynomial* p = calloc(n, sizeof(Polynomial));
-  if (p == NULL) return -1;
-
-  for (size_t i = 0; i < n; i++)
-    for (size_t j = 0; j < FIELD_ELEMENTS_PER_BLOB; j++)
-      bytes_to_bls_field(&p[i][j], &blobs[i * FIELD_ELEMENTS_PER_BLOB * 32 + j * 32]);
-
   KZGProof f;
-  C_KZG_RET ret = compute_aggregate_kzg_proof(&f, p, n, s);
+  C_KZG_RET ret;
-
+  ret = compute_aggregate_kzg_proof(&f, blobs, n, s);
-  free(p);
+  if (ret != C_KZG_OK) return -1;
-  if (ret != C_KZG_OK) return ret;
-
   bytes_from_g1(out, &f);
   return C_KZG_OK;
 }

bindings/python/ckzg.c

@@ -42,12 +42,7 @@ static PyObject* blob_to_kzg_commitment_wrap(PyObject *self, PyObject *args) {
   if (PyBytes_Size(b) != 32 * 4096)
     return PyErr_Format(PyExc_ValueError, "expected 32 * 4096 bytes");
 
-  uint8_t* bytes = (uint8_t*)PyBytes_AsString(b);
+  uint8_t* blob = (uint8_t*)PyBytes_AsString(b);
-
-  BLSFieldElement blob[4096];
-
-  for (int i = 0; i < 4096; i++)
-    bytes_to_bls_field(&blob[i], &bytes[i * 32]);
 
   KZGCommitment *k = (KZGCommitment*)malloc(sizeof(KZGCommitment));
 
@@ -71,31 +66,20 @@ static PyObject* compute_aggregate_kzg_proof_wrap(PyObject *self, PyObject *args
     return PyErr_Format(PyExc_ValueError, "expected a multiple of 32 * 4096 bytes");
   n = n / 32 / 4096;
 
-  Polynomial* blobs = calloc(n, sizeof(Polynomial));
+  Blob* blobs = (Blob*)PyBytes_AsString(b);
-
-  if (blobs == NULL) return PyErr_NoMemory();
-
-  uint8_t* bytes = (uint8_t*)PyBytes_AsString(b);
-
-  for (Py_ssize_t i = 0; i < n; i++)
-    for (int j = 0; j < 4096; j++)
-      bytes_to_bls_field(&blobs[i][j], &bytes[i * 32 * 4096 + j * 32]);
 
   KZGProof *k = (KZGProof*)malloc(sizeof(KZGProof));
 
   if (k == NULL) {
-    free(blobs);
     return PyErr_NoMemory();
   }
 
   if (compute_aggregate_kzg_proof(k, blobs, n,
         PyCapsule_GetPointer(s, "KZGSettings")) != C_KZG_OK) {
     free(k);
-    free(blobs);
     return PyErr_Format(PyExc_RuntimeError, "compute_aggregate_kzg_proof failed");
   }
 
-  free(blobs);
   return PyCapsule_New(k, "G1", free_G1);
 }
 
@@ -118,26 +102,18 @@ static PyObject* verify_aggregate_kzg_proof_wrap(PyObject *self, PyObject *args)
   if (PySequence_Length(c) != n)
     return PyErr_Format(PyExc_ValueError, "expected same number of commitments as polynomials");
 
-  Polynomial* blobs = calloc(n, sizeof(Polynomial));
-
-  if (blobs == NULL) return PyErr_NoMemory();
-
   KZGCommitment* commitments = calloc(n, sizeof(KZGCommitment));
 
   if (commitments == NULL) {
-    free(blobs);
     return PyErr_NoMemory();
   }
 
-  uint8_t* bytes = (uint8_t*)PyBytes_AsString(b);
+  Blob* blobs = (Blob*)PyBytes_AsString(b);
 
   for (Py_ssize_t i = 0; i < n; i++) {
-    for (int j = 0; j < 4096; j++)
-      bytes_to_bls_field(&blobs[i][j], &bytes[i * 32 * 4096 + j * 32]);
     e = PySequence_GetItem(c, i);
     if (!PyCapsule_IsValid(e, "G1")) {
       free(commitments);
-      free(blobs);
       return PyErr_Format(PyExc_ValueError, "expected G1 capsules");
     }
     memcpy(&commitments[i], PyCapsule_GetPointer(e, "G1"), sizeof(KZGCommitment));
@@ -150,12 +126,10 @@ static PyObject* verify_aggregate_kzg_proof_wrap(PyObject *self, PyObject *args)
         PyCapsule_GetPointer(p, "G1"),
         PyCapsule_GetPointer(s, "KZGSettings")) != C_KZG_OK) {
     free(commitments);
-    free(blobs);
     return PyErr_Format(PyExc_RuntimeError, "verify_aggregate_kzg_proof failed");
   }
 
   free(commitments);
-  free(blobs);
   if (out) Py_RETURN_TRUE; else Py_RETURN_FALSE;
 }
 

src/c_kzg_4844.c

@@ -764,7 +764,7 @@ C_KZG_RET bytes_to_g1(g1_t* out, const uint8_t bytes[48]) {
   return C_KZG_OK;
 }
 
-void bytes_from_bls_field(uint8_t out[32], const BLSFieldElement *in) {
+static void bytes_from_bls_field(uint8_t out[32], const BLSFieldElement *in) {
   blst_scalar_from_fr((blst_scalar*)out, in);
 }
 
@@ -825,7 +825,7 @@ void free_trusted_setup(KZGSettings *s) {
 
 static void compute_powers(fr_t out[], uint64_t n) {
     uint64_t i = 0;
-    out[++i] = fr_one;
+    out[i++] = fr_one;
     while (++i < n) fr_mul(&out[i], &out[i-1], &out[1]);
 }
 
@@ -835,7 +835,7 @@ void bytes_to_bls_field(BLSFieldElement *out, const uint8_t bytes[32]) {
   blst_fr_from_scalar(out, &tmp);
 }
 
-static void vector_lincomb(Polynomial out, const Polynomial vectors[], const fr_t scalars[], uint64_t n) {
+static void poly_lincomb(Polynomial out, const Polynomial vectors[], const fr_t scalars[], uint64_t n) {
   fr_t tmp;
   uint64_t i, j;
   for (j = 0; j < FIELD_ELEMENTS_PER_BLOB; j++)
@@ -904,8 +904,19 @@ static void g1_lincomb(g1_t *out, const g1_t *p, const fr_t *coeffs, const uint6
     }
 }
 
-void blob_to_kzg_commitment(KZGCommitment *out, const Polynomial blob, const KZGSettings *s) {
+static void poly_to_kzg_commitment(KZGCommitment *out, const Polynomial p, const KZGSettings *s) {
-  g1_lincomb(out, s->g1_values, blob, FIELD_ELEMENTS_PER_BLOB);
+  g1_lincomb(out, s->g1_values, p, FIELD_ELEMENTS_PER_BLOB);
+}
+
+static void poly_from_blob(Polynomial p, const Blob blob) {
+  for (size_t i = 0; i < FIELD_ELEMENTS_PER_BLOB; i++)
+    bytes_to_bls_field(&p[i], &blob[i * BYTES_PER_FIELD_ELEMENT]);
+}
+
+void blob_to_kzg_commitment(KZGCommitment *out, const Blob blob, const KZGSettings *s) {
+  Polynomial p;
+  poly_from_blob(p, blob);
+  poly_to_kzg_commitment(out, p, s);
 }
 
 /**
@@ -1059,20 +1070,29 @@ static void hash(uint8_t md[32], uint8_t input[], size_t n) {
   sha256_final(md, &ctx);
 }
 
+static void bytes_of_uint64(uint8_t out[8], uint64_t n) {
+  for (int i = 0; i < 8; i++) {
+    out[i] = n & 0xFF;
+    n >>= 8;
+  }
+}
+
 static C_KZG_RET hash_to_bytes(uint8_t out[32],
-    const uint8_t *initializer, const Polynomial polys[], const KZGCommitment comms[], size_t n) {
+    const Polynomial polys[], const KZGCommitment comms[], uint64_t n) {
   size_t i; uint64_t j;
-  size_t ni = initializer == NULL ? 0 : 32;
+  size_t ni = 32; // len(FIAT_SHAMIR_PROTOCOL_DOMAIN) + 8 + 8
   size_t np = ni + n * FIELD_ELEMENTS_PER_BLOB * 32;
 
   uint8_t* bytes = calloc(np + n * 48, sizeof(uint8_t));
   if (bytes == NULL) return C_KZG_MALLOC;
 
-  if (ni) memcpy(bytes, initializer, ni);
+  memcpy(bytes, FIAT_SHAMIR_PROTOCOL_DOMAIN, 16);
+  bytes_of_uint64(&bytes[16], n);
+  bytes_of_uint64(&bytes[16 + 8], FIELD_ELEMENTS_PER_BLOB);
 
   for (i = 0; i < n; i++)
     for (j = 0; j < FIELD_ELEMENTS_PER_BLOB; j++)
-      bytes_from_bls_field(&bytes[ni + i * 32], &polys[i][j]);
+      bytes_from_bls_field(&bytes[ni + i * BYTES_PER_FIELD_ELEMENT], &polys[i][j]);
 
   for (i = 0; i < n; i++)
     bytes_from_g1(&bytes[np + i * 48], &comms[i]);
@@ -1083,21 +1103,25 @@ static C_KZG_RET hash_to_bytes(uint8_t out[32],
   return C_KZG_OK;
 }
 
-static C_KZG_RET compute_aggregated_poly_and_commitment(Polynomial poly_out, KZGCommitment *comm_out, uint8_t hash_out[32],
+static C_KZG_RET compute_aggregated_poly_and_commitment(Polynomial poly_out, KZGCommitment *comm_out, BLSFieldElement *chal_out,
-    const Polynomial blobs[],
+    const Polynomial polys[],
     const KZGCommitment kzg_commitments[],
     size_t n) {
   BLSFieldElement* r_powers = calloc(n, sizeof(BLSFieldElement));
   if (r_powers == NULL) return C_KZG_MALLOC;
 
   C_KZG_RET ret;
-  ret = hash_to_bytes(hash_out, NULL, blobs, kzg_commitments, n);
+  uint8_t* hash = calloc(32, sizeof(uint8_t));
-  if (ret != C_KZG_OK) { free(r_powers); return ret; }
+  if (hash == NULL) { free(r_powers); return C_KZG_MALLOC; }
-  bytes_to_bls_field(&r_powers[1], hash_out);
+  ret = hash_to_bytes(hash, polys, kzg_commitments, n);
+  if (ret != C_KZG_OK) { free(r_powers); free(hash); return ret; }
+  bytes_to_bls_field(&r_powers[1], hash);
+  free(hash);
 
   compute_powers(r_powers, n);
+  fr_mul(chal_out, &r_powers[1], &r_powers[n - 1]);
 
-  vector_lincomb(poly_out, blobs, r_powers, n);
+  poly_lincomb(poly_out, polys, r_powers, n);
 
   g1_lincomb(comm_out, kzg_commitments, r_powers, n);
 
@@ -1106,45 +1130,53 @@ static C_KZG_RET compute_aggregated_poly_and_commitment(Polynomial poly_out, KZG
 }
 
 C_KZG_RET compute_aggregate_kzg_proof(KZGProof *out,
-    const Polynomial blobs[],
+    const Blob blobs[],
     size_t n,
     const KZGSettings *s) {
   KZGCommitment* commitments = calloc(n, sizeof(KZGCommitment));
   if (commitments == NULL) return C_KZG_MALLOC;
 
-  for (size_t i = 0; i < n; i++)
+  Polynomial* polys = calloc(n, sizeof(Polynomial));
-    blob_to_kzg_commitment(&commitments[i], blobs[i], s);
+  if (polys == NULL) { free(commitments); return C_KZG_MALLOC; }
+
+  for (size_t i = 0; i < n; i++) {
+    poly_from_blob(polys[i], blobs[i]);
+    poly_to_kzg_commitment(&commitments[i], polys[i], s);
+  }
 
   Polynomial aggregated_poly;
   KZGCommitment aggregated_poly_commitment;
+  BLSFieldElement evaluation_challenge;
   C_KZG_RET ret;
-  uint8_t hash[32];
+  ret = compute_aggregated_poly_and_commitment(aggregated_poly, &aggregated_poly_commitment, &evaluation_challenge, polys, commitments, n);
-  ret = compute_aggregated_poly_and_commitment(aggregated_poly, &aggregated_poly_commitment, hash, blobs, commitments, n);
   free(commitments);
+  free(polys);
   if (ret != C_KZG_OK) return ret;
 
-  TRY(hash_to_bytes(hash, hash, &aggregated_poly, &aggregated_poly_commitment, 1));
+  return compute_kzg_proof(out, aggregated_poly, &evaluation_challenge, s);
-  BLSFieldElement x;
-  bytes_to_bls_field(&x, hash);
-
-  return compute_kzg_proof(out, aggregated_poly, &x, s);
 }
 
 C_KZG_RET verify_aggregate_kzg_proof(bool *out,
-    const Polynomial blobs[],
+    const Blob blobs[],
     const KZGCommitment expected_kzg_commitments[],
     size_t n,
     const KZGProof *kzg_aggregated_proof,
     const KZGSettings *s) {
+  Polynomial* polys = calloc(n, sizeof(Polynomial));
+  if (polys == NULL) return C_KZG_MALLOC;
+  for (size_t i = 0; i < n; i++)
+    poly_from_blob(polys[i], blobs[i]);
+
   Polynomial aggregated_poly;
   KZGCommitment aggregated_poly_commitment;
-  uint8_t hash[32];
+  BLSFieldElement evaluation_challenge;
-  TRY(compute_aggregated_poly_and_commitment(aggregated_poly, &aggregated_poly_commitment, hash, blobs, expected_kzg_commitments, n));
+  C_KZG_RET ret;
+  ret = compute_aggregated_poly_and_commitment(aggregated_poly, &aggregated_poly_commitment, &evaluation_challenge, polys, expected_kzg_commitments, n);
+  free(polys);
+  if (ret != C_KZG_OK) return ret;
 
-  BLSFieldElement x, y;
+  BLSFieldElement y;
-  TRY(hash_to_bytes(hash, hash, &aggregated_poly, &aggregated_poly_commitment, 1));
+  TRY(evaluate_polynomial_in_evaluation_form(&y, aggregated_poly, &evaluation_challenge, s));
-  bytes_to_bls_field(&x, hash);
-  TRY(evaluate_polynomial_in_evaluation_form(&y, aggregated_poly, &x, s));
 
-  return verify_kzg_proof(out, &aggregated_poly_commitment, &x, &y, kzg_aggregated_proof, s);
+  return verify_kzg_proof(out, &aggregated_poly_commitment, &evaluation_challenge, &y, kzg_aggregated_proof, s);
 }

src/c_kzg_4844.h

@@ -38,6 +38,8 @@ extern "C" {
 #define BYTES_PER_PROOF 48
 #define BYTES_PER_FIELD 32
 #define FIELD_ELEMENTS_PER_BLOB 4096
+#define BYTES_PER_FIELD_ELEMENT 32
+static const uint8_t FIAT_SHAMIR_PROTOCOL_DOMAIN[] = {70, 83, 66, 76, 79, 66, 86, 69, 82, 73, 70, 89, 95, 86, 49, 95}; // "FSBLOBVERIFY_V1_"
 
 typedef blst_p1 g1_t;         /**< Internal G1 group element type */
 typedef blst_p2 g2_t;         /**< Internal G2 group element type */
@@ -46,6 +48,7 @@ typedef blst_fr fr_t;         /**< Internal Fr field element type */
 typedef g1_t KZGCommitment;
 typedef g1_t KZGProof;
 typedef fr_t BLSFieldElement;
+typedef uint8_t Blob[BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB];
 typedef BLSFieldElement Polynomial[FIELD_ELEMENTS_PER_BLOB];
 
 /**
@@ -62,18 +65,6 @@ typedef enum {
     C_KZG_MALLOC,  /**< Could not allocate memory */
 } C_KZG_RET;
 
-/**
- * KZGCommitment and KZGProof can be recovered as 48 bytes
- */
-void bytes_from_g1(uint8_t out[48], const g1_t*);
-C_KZG_RET bytes_to_g1(g1_t* out, const uint8_t[48]);
-
-/**
- * BLSFieldElements can be recovered as 32 bytes
- */
-void bytes_from_bls_field(uint8_t out[32], const BLSFieldElement*);
-void bytes_to_bls_field(BLSFieldElement *out, const uint8_t bytes[32]);
-
 /**
  * Stores the setup and parameters needed for performing FFTs.
  */
@@ -97,6 +88,11 @@ typedef struct {
  * Interface functions
  */
 
+C_KZG_RET bytes_to_g1(g1_t* out, const uint8_t in[48]);
+void bytes_from_g1(uint8_t out[48], const g1_t *in);
+
+void bytes_to_bls_field(BLSFieldElement *out, const uint8_t in[BYTES_PER_FIELD_ELEMENT]);
+
 C_KZG_RET load_trusted_setup(KZGSettings *out,
     FILE *in);
 
@@ -104,19 +100,19 @@ void free_trusted_setup(
     KZGSettings *s);
 
 C_KZG_RET compute_aggregate_kzg_proof(KZGProof *out,
-    const Polynomial blobs[],
+    const Blob blobs[],
     size_t n,
     const KZGSettings *s);
 
 C_KZG_RET verify_aggregate_kzg_proof(bool *out,
-    const Polynomial blobs[],
+    const Blob blobs[],
     const KZGCommitment expected_kzg_commitments[],
     size_t n,
     const KZGProof *kzg_aggregated_proof,
     const KZGSettings *s);
 
 void blob_to_kzg_commitment(KZGCommitment *out,
-    const Polynomial blob,
+    const Blob blob,
     const KZGSettings *s);
 
 C_KZG_RET verify_kzg_proof(bool *out,