Merge #726

726: Remove buffer usage r=D4nte a=D4nte

This only removes it from the code we wrote.

- `ts-proto` uses `Buffer`, the idea would be to review whether we can use the new `protons`
- Some dependencies are likely to use `Buffer`, will need review.

Co-authored-by: Franck Royer <franck@status.im>

package.json

@@ -42,10 +42,11 @@
     "examples:pretest": "for d in examples/*/; do (cd $d && npm install); done",
     "nim-waku:build": "(cd nim-waku; NIMFLAGS=\"-d:chronicles_colors=off -d:chronicles_sinks=textlines -d:chronicles_log_level=TRACE\" make -j$(nproc --all 2>/dev/null || echo 2) wakunode2)",
     "nim-waku:force-build": "(cd nim-waku && rm -rf ./build/ ./vendor && make -j$(nproc --all 2>/dev/null || echo 2) update) && run-s nim-waku:build",
-    "test": "run-s build test:*",
+    "test": "run-s test:*",
     "test:lint": "eslint src --ext .ts",
     "test:prettier": "prettier \"src/**/*.ts\" \"./*.json\" \"*.conf.js\" \".github/**/*.yml\" --list-different",
     "test:spelling": "cspell \"{README.md,.github/*.md,guides/*.md,src/**/*.ts}\"",
+    "test:tsc": "tsc -p tsconfig.dev.json",
     "test:unit": "nyc --silent mocha",
     "test:karma": "karma start",
     "examples:test": "run-s examples:pretest; for d in examples/*/; do (cd $d && npm test;); done",

src/lib/waku_message/symmetric.ts

@@ -7,29 +7,29 @@ export const TagSize = 16;
 const Algorithm = { name: "AES-GCM", length: 128 };
 
 export async function encrypt(
-  iv: Buffer | Uint8Array,
+  iv: Uint8Array,
-  key: Buffer,
+  key: Uint8Array,
-  clearText: Buffer
+  clearText: Uint8Array
-): Promise<Buffer> {
+): Promise<Uint8Array> {
   return getSubtle()
     .importKey("raw", key, Algorithm, false, ["encrypt"])
     .then((cryptoKey) =>
       getSubtle().encrypt({ iv, ...Algorithm }, cryptoKey, clearText)
     )
-    .then(Buffer.from);
+    .then((cipher) => new Uint8Array(cipher));
 }
 
 export async function decrypt(
-  iv: Buffer,
+  iv: Uint8Array,
-  key: Buffer,
+  key: Uint8Array,
-  cipherText: Buffer
+  cipherText: Uint8Array
-): Promise<Buffer> {
+): Promise<Uint8Array> {
   return getSubtle()
     .importKey("raw", key, Algorithm, false, ["decrypt"])
     .then((cryptoKey) =>
       getSubtle().decrypt({ iv, ...Algorithm }, cryptoKey, cipherText)
     )
-    .then(Buffer.from);
+    .then((clear) => new Uint8Array(clear));
 }
 
 export function generateIv(): Uint8Array {

src/lib/waku_message/version_1.ts

@@ -1,7 +1,6 @@
-import { Buffer } from "buffer";
-
 import * as secp from "@noble/secp256k1";
 import { keccak256 } from "js-sha3";
+import { concat } from "uint8arrays/concat";
 
 import { randomBytes } from "../crypto";
 import { hexToBytes } from "../utils";
@@ -17,6 +16,11 @@ const SignatureLength = 65;
 
 export const PrivateKeySize = 32;
 
+export type Signature = {
+  signature: Uint8Array;
+  publicKey: Uint8Array | undefined;
+};
+
 /**
  * Encode the payload pre-encryption.
  *
@@ -30,14 +34,14 @@ export async function clearEncode(
   messagePayload: Uint8Array,
   sigPrivKey?: Uint8Array
 ): Promise<{ payload: Uint8Array; sig?: Signature }> {
-  let envelope = Buffer.from([0]); // No flags
+  let envelope = new Uint8Array([0]); // No flags
   envelope = addPayloadSizeField(envelope, messagePayload);
-  envelope = Buffer.concat([envelope, Buffer.from(messagePayload)]);
+  envelope = concat([envelope, messagePayload]);
 
   // Calculate padding:
   let rawSize =
     FlagsLength +
-    getSizeOfPayloadSizeField(messagePayload) +
+    computeSizeOfPayloadSizeField(messagePayload) +
     messagePayload.length;
 
   if (sigPrivKey) {
@@ -46,29 +50,26 @@ export async function clearEncode(
 
   const remainder = rawSize % PaddingTarget;
   const paddingSize = PaddingTarget - remainder;
-  const pad = Buffer.from(randomBytes(paddingSize));
+  const pad = randomBytes(paddingSize);
 
   if (!validateDataIntegrity(pad, paddingSize)) {
     throw new Error("failed to generate random padding of size " + paddingSize);
   }
 
-  envelope = Buffer.concat([envelope, pad]);
+  envelope = concat([envelope, pad]);
 
   let sig;
   if (sigPrivKey) {
     envelope[0] |= IsSignedMask;
     const hash = keccak256(envelope);
-    const [signature, recid] = await secp.sign(hash, sigPrivKey, {
+    const [hexSignature, recid] = await secp.sign(hash, sigPrivKey, {
       recovered: true,
       der: false,
     });
-    envelope = Buffer.concat([
+    const bytesSignature = hexToBytes(hexSignature);
-      envelope,
+    envelope = concat([envelope, bytesSignature, [recid]]);
-      hexToBytes(signature),
-      Buffer.from([recid]),
-    ]);
     sig = {
-      signature: Buffer.from(signature),
+      signature: bytesSignature,
       publicKey: getPublicKey(sigPrivKey),
     };
   }
@@ -76,35 +77,27 @@ export async function clearEncode(
   return { payload: envelope, sig };
 }
 
-export type Signature = {
-  signature: Uint8Array;
-  publicKey: Uint8Array | undefined;
-};
-
 /**
  * Decode a decrypted payload.
  *
  * @internal
  */
 export function clearDecode(
-  message: Uint8Array | Buffer
+  message: Uint8Array
 ): { payload: Uint8Array; sig?: Signature } | undefined {
-  const buf = Buffer.from(message);
+  const sizeOfPayloadSizeField = getSizeOfPayloadSizeField(message);
-  let start = 1;
-  let sig;
-
-  const sizeOfPayloadSizeField = buf.readUIntLE(0, 1) & FlagMask;
-
   if (sizeOfPayloadSizeField === 0) return;
 
-  const payloadSize = buf.readUIntLE(start, sizeOfPayloadSizeField);
+  const payloadSize = getPayloadSize(message, sizeOfPayloadSizeField);
-  start += sizeOfPayloadSizeField;
+  const payloadStart = 1 + sizeOfPayloadSizeField;
-  const payload = buf.slice(start, start + payloadSize);
+  const payload = message.slice(payloadStart, payloadStart + payloadSize);
 
-  const isSigned = (buf.readUIntLE(0, 1) & IsSignedMask) == IsSignedMask;
+  const isSigned = isMessageSigned(message);
+
+  let sig;
   if (isSigned) {
-    const signature = getSignature(buf);
+    const signature = getSignature(message);
-    const hash = getHash(buf, isSigned);
+    const hash = getHash(message, isSigned);
     const publicKey = ecRecoverPubKey(hash, signature);
     sig = { signature, publicKey };
   }
@@ -112,31 +105,58 @@ export function clearDecode(
   return { payload, sig };
 }
 
+function getSizeOfPayloadSizeField(message: Uint8Array): number {
+  const messageDataView = new DataView(message.buffer);
+  return messageDataView.getUint8(0) & FlagMask;
+}
+
+function getPayloadSize(
+  message: Uint8Array,
+  sizeOfPayloadSizeField: number
+): number {
+  let payloadSizeBytes = message.slice(1, 1 + sizeOfPayloadSizeField);
+  // int 32 == 4 bytes
+  if (sizeOfPayloadSizeField < 4) {
+    // If less than 4 bytes pad right (Little Endian).
+    payloadSizeBytes = concat(
+      [payloadSizeBytes, new Uint8Array(4 - sizeOfPayloadSizeField)],
+      4
+    );
+  }
+  const payloadSizeDataView = new DataView(payloadSizeBytes.buffer);
+  return payloadSizeDataView.getInt32(0, true);
+}
+
+function isMessageSigned(message: Uint8Array): boolean {
+  const messageDataView = new DataView(message.buffer);
+  return (messageDataView.getUint8(0) & IsSignedMask) == IsSignedMask;
+}
+
 /**
  * Proceed with Asymmetric encryption of the data as per [26/WAKU-PAYLOAD](https://rfc.vac.dev/spec/26/).
  * The data MUST be flags | payload-length | payload | [signature].
- * The returned result can be set to `WakuMessage.payload`.
+ * The returned result  can be set to `WakuMessage.payload`.
  *
  * @internal
  */
 export async function encryptAsymmetric(
-  data: Uint8Array | Buffer,
+  data: Uint8Array,
-  publicKey: Uint8Array | Buffer | string
+  publicKey: Uint8Array | string
 ): Promise<Uint8Array> {
-  return ecies.encrypt(Buffer.from(hexToBytes(publicKey)), Buffer.from(data));
+  return ecies.encrypt(hexToBytes(publicKey), data);
 }
 
 /**
  * Proceed with Asymmetric decryption of the data as per [26/WAKU-PAYLOAD](https://rfc.vac.dev/spec/26/).
- * The return data is expect to be flags | payload-length | payload | [signature].
+ * The returned data is expected to be `flags | payload-length | payload | [signature]`.
  *
  * @internal
  */
 export async function decryptAsymmetric(
-  payload: Uint8Array | Buffer,
+  payload: Uint8Array,
-  privKey: Uint8Array | Buffer
+  privKey: Uint8Array
 ): Promise<Uint8Array> {
-  return ecies.decrypt(Buffer.from(privKey), Buffer.from(payload));
+  return ecies.decrypt(privKey, payload);
 }
 
 /**
@@ -149,18 +169,14 @@ export async function decryptAsymmetric(
  * @internal
  */
 export async function encryptSymmetric(
-  data: Uint8Array | Buffer,
+  data: Uint8Array,
-  key: Uint8Array | Buffer | string
+  key: Uint8Array | string
 ): Promise<Uint8Array> {
   const iv = symmetric.generateIv();
 
   // Returns `cipher | tag`
-  const cipher = await symmetric.encrypt(
+  const cipher = await symmetric.encrypt(iv, hexToBytes(key), data);
-    iv,
+  return concat([cipher, iv]);
-    Buffer.from(hexToBytes(key)),
-    Buffer.from(data)
-  );
-  return Buffer.concat([cipher, Buffer.from(iv)]);
 }
 
 /**
@@ -173,15 +189,14 @@ export async function encryptSymmetric(
  * @internal
  */
 export async function decryptSymmetric(
-  payload: Uint8Array | Buffer,
+  payload: Uint8Array,
-  key: Uint8Array | Buffer | string
+  key: Uint8Array | string
 ): Promise<Uint8Array> {
-  const data = Buffer.from(payload);
+  const ivStart = payload.length - symmetric.IvSize;
-  const ivStart = data.length - symmetric.IvSize;
+  const cipher = payload.slice(0, ivStart);
-  const cipher = data.slice(0, ivStart);
+  const iv = payload.slice(ivStart);
-  const iv = data.slice(ivStart);
 
-  return symmetric.decrypt(iv, Buffer.from(hexToBytes(key)), cipher);
+  return symmetric.decrypt(iv, hexToBytes(key), cipher);
 }
 
 /**
@@ -204,19 +219,20 @@ export function generateSymmetricKey(): Uint8Array {
  * Return the public key for the given private key, to be used for asymmetric
  * encryption.
  */
-export function getPublicKey(privateKey: Uint8Array | Buffer): Uint8Array {
+export function getPublicKey(privateKey: Uint8Array): Uint8Array {
   return secp.getPublicKey(privateKey, false);
 }
 
 /**
  * Computes the flags & auxiliary-field as per [26/WAKU-PAYLOAD](https://rfc.vac.dev/spec/26/).
  */
-function addPayloadSizeField(msg: Buffer, payload: Uint8Array): Buffer {
+function addPayloadSizeField(msg: Uint8Array, payload: Uint8Array): Uint8Array {
-  const fieldSize = getSizeOfPayloadSizeField(payload);
+  const fieldSize = computeSizeOfPayloadSizeField(payload);
-  let field = Buffer.alloc(4);
+  let field = new Uint8Array(4);
-  field.writeUInt32LE(payload.length, 0);
+  const fieldDataView = new DataView(field.buffer);
+  fieldDataView.setUint32(0, payload.length, true);
   field = field.slice(0, fieldSize);
-  msg = Buffer.concat([msg, field]);
+  msg = concat([msg, field]);
   msg[0] |= fieldSize;
   return msg;
 }
@@ -224,7 +240,7 @@ function addPayloadSizeField(msg: Buffer, payload: Uint8Array): Buffer {
 /**
  * Returns the size of the auxiliary-field which in turns contains the payload size
  */
-function getSizeOfPayloadSizeField(payload: Uint8Array): number {
+function computeSizeOfPayloadSizeField(payload: Uint8Array): number {
   let s = 1;
   for (let i = payload.length; i >= 256; i /= 256) {
     s++;
@@ -240,16 +256,14 @@ function validateDataIntegrity(
     return false;
   }
 
-  return !(
+  return expectedSize <= 3 || value.findIndex((i) => i !== 0) !== -1;
-    expectedSize > 3 && Buffer.from(value).equals(Buffer.alloc(value.length))
-  );
 }
 
-function getSignature(message: Buffer): Buffer {
+function getSignature(message: Uint8Array): Uint8Array {
   return message.slice(message.length - SignatureLength, message.length);
 }
 
-function getHash(message: Buffer, isSigned: boolean): string {
+function getHash(message: Uint8Array, isSigned: boolean): string {
   if (isSigned) {
     return keccak256(message.slice(0, message.length - SignatureLength));
   }
@@ -258,9 +272,10 @@ function getHash(message: Buffer, isSigned: boolean): string {
 
 function ecRecoverPubKey(
   messageHash: string,
-  signature: Buffer
+  signature: Uint8Array
 ): Uint8Array | undefined {
-  const recovery = signature.slice(64).readIntBE(0, 1);
+  const recoveryDataView = new DataView(signature.slice(64).buffer);
+  const recovery = recoveryDataView.getUint8(0);
   const _signature = secp.Signature.fromCompact(signature.slice(0, 64));
 
   return secp.recoverPublicKey(