Merge #634

634: Replace conversion functions with `uint8arrays` r=D4nte a=D4nte

Resolves #603

Co-authored-by: Franck Royer <franck@status.im>

CHANGELOG.md

@@ -10,11 +10,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Examples: Add Relay JavaScript example.
+- **Breaking**: Moved utf-8 conversion functions to `utils`.
 
 ### Fixed
 
 - Replace Base 64 buggy conversion functions with `uint8arrays`.
 
+### Removed
+
+- **Breaking**: Removed `equalByteArrays`, use `uint8arrays/equals` instead.
+  See changes in `eth-pm` example.
+- **Breaking**: Removed deprecated utils functions.
+
 ## [0.19.2] - 2022-03-21
 
 ### Fixed

examples/eth-pm-wallet-encryption/src/crypto.ts

@@ -1,8 +1,9 @@
 import "@ethersproject/shims";
 
 import { PublicKeyMessage } from "./messaging/wire";
-import { hexToBytes, equalByteArrays, bytesToHex } from "js-waku/lib/utils";
+import { hexToBytes, bytesToHex } from "js-waku/lib/utils";
 import * as sigUtil from "eth-sig-util";
+import { equals } from "uint8arrays/equals";
 
 /**
  * Sign the encryption public key with Web3. This can then be published to let other
@@ -97,5 +98,5 @@ export function validatePublicKeyMessage(msg: PublicKeyMessage): boolean {
   console.log("Recovered", recovered);
   console.log("ethAddress", "0x" + bytesToHex(msg.ethAddress));
 
-  return equalByteArrays(recovered, msg.ethAddress);
+  return equals(hexToBytes(recovered), msg.ethAddress);
 }

examples/eth-pm-wallet-encryption/src/waku.ts

@@ -3,7 +3,8 @@ import { Waku, WakuMessage } from "js-waku";
 import { PrivateMessage, PublicKeyMessage } from "./messaging/wire";
 import { validatePublicKeyMessage } from "./crypto";
 import { Message } from "./messaging/Messages";
-import { bytesToHex, equalByteArrays } from "js-waku/lib/utils";
+import { bytesToHex, hexToBytes } from "js-waku/lib/utils";
+import { equals } from "uint8arrays/equals";
 
 export const PublicKeyContentTopic =
   "/eth-pm-wallet/1/encryption-public-key/proto";
@@ -36,7 +37,8 @@ export function handlePublicKeyMessage(
   if (!msg.payload) return;
   const publicKeyMsg = PublicKeyMessage.decode(msg.payload);
   if (!publicKeyMsg) return;
-  if (myAddress && equalByteArrays(publicKeyMsg.ethAddress, myAddress)) return;
+  if (myAddress && equals(publicKeyMsg.ethAddress, hexToBytes(myAddress)))
+    return;
 
   const res = validatePublicKeyMessage(publicKeyMsg);
   console.log("Is Public Key Message valid?", res);
@@ -77,7 +79,7 @@ export async function handlePrivateMessage(
     console.log("Failed to decode Private Message");
     return;
   }
-  if (!equalByteArrays(privateMessage.toAddress, address)) return;
+  if (!equals(privateMessage.toAddress, hexToBytes(address))) return;
 
   const timestamp = wakuMsg.timestamp ? wakuMsg.timestamp : new Date();
 

examples/eth-pm/src/crypto.ts

@@ -1,11 +1,12 @@
 import "@ethersproject/shims";
 
 import { PublicKeyMessage } from "./messaging/wire";
-import { hexToBytes, equalByteArrays, bytesToHex } from "js-waku/lib/utils";
+import { hexToBytes, bytesToHex } from "js-waku/lib/utils";
 import { generatePrivateKey, getPublicKey } from "js-waku";
 import * as sigUtil from "eth-sig-util";
 import { PublicKeyContentTopic } from "./waku";
 import { keccak256 } from "ethers/lib/utils";
+import { equals } from "uint8arrays/equals";
 
 export const PublicKeyMessageEncryptionKey = hexToBytes(
   keccak256(Buffer.from(PublicKeyContentTopic, "utf-8"))
@@ -118,5 +119,5 @@ export function validatePublicKeyMessage(msg: PublicKeyMessage): boolean {
   console.log("Recovered", recovered);
   console.log("ethAddress", "0x" + bytesToHex(msg.ethAddress));
 
-  return equalByteArrays(recovered, msg.ethAddress);
+  return equals(hexToBytes(recovered), msg.ethAddress);
 }

examples/eth-pm/src/waku.ts

@@ -3,7 +3,8 @@ import { Waku, WakuMessage } from "js-waku";
 import { PrivateMessage, PublicKeyMessage } from "./messaging/wire";
 import { validatePublicKeyMessage } from "./crypto";
 import { Message } from "./messaging/Messages";
-import { bytesToHex, equalByteArrays } from "js-waku/lib/utils";
+import { bytesToHex, hexToBytes } from "js-waku/lib/utils";
+import { equals } from "uint8arrays/equals";
 
 export const PublicKeyContentTopic = "/eth-pm/1/public-key/proto";
 export const PrivateMessageContentTopic = "/eth-pm/1/private-message/proto";
@@ -34,7 +35,8 @@ export function handlePublicKeyMessage(
   if (!msg.payload) return;
   const publicKeyMsg = PublicKeyMessage.decode(msg.payload);
   if (!publicKeyMsg) return;
-  if (myAddress && equalByteArrays(publicKeyMsg.ethAddress, myAddress)) return;
+  if (myAddress && equals(publicKeyMsg.ethAddress, hexToBytes(myAddress)))
+    return;
 
   const res = validatePublicKeyMessage(publicKeyMsg);
   console.log("Is Public Key Message valid?", res);
@@ -62,7 +64,7 @@ export async function handlePrivateMessage(
     console.log("Failed to decode Private Message");
     return;
   }
-  if (!equalByteArrays(privateMessage.toAddress, address)) return;
+  if (!equals(privateMessage.toAddress, hexToBytes(address))) return;
 
   const timestamp = wakuMsg.timestamp ? wakuMsg.timestamp : new Date();
 

src/index.ts

@@ -3,8 +3,6 @@ export * as discovery from "./lib/discovery";
 
 export * as enr from "./lib/enr";
 
-export * as utf8 from "./lib/utf8";
-
 export * as utils from "./lib/utils";
 
 export * as waku from "./lib/waku";

src/lib/discovery/dns_over_https.ts

@@ -6,7 +6,7 @@ import {
   query,
 } from "dns-query";
 
-import { bytesToUtf8 } from "../utf8";
+import { bytesToUtf8 } from "../utils";
 
 import { DnsClient } from "./dns";
 

src/lib/discovery/enrtree.ts

@@ -5,8 +5,7 @@ import { ecdsaVerify } from "secp256k1";
 import { fromString } from "uint8arrays/from-string";
 
 import { ENR } from "../enr";
-import { utf8ToBytes } from "../utf8";
+import { keccak256Buf, utf8ToBytes } from "../utils";
-import { keccak256Buf } from "../utils";
 
 export type ENRRootValues = {
   eRoot: string;

src/lib/enr/enr.spec.ts

@@ -2,8 +2,7 @@ import { assert, expect } from "chai";
 import { Multiaddr } from "multiaddr";
 import PeerId from "peer-id";
 
-import { utf8ToBytes } from "../utf8";
+import { bytesToHex, hexToBytes, utf8ToBytes } from "../utils";
-import { bytesToHex, hexToBytes } from "../utils";
 
 import { ERR_INVALID_ID } from "./constants";
 import { ENR } from "./enr";

src/lib/enr/enr.ts

@@ -9,8 +9,7 @@ import { fromString } from "uint8arrays/from-string";
 import { toString } from "uint8arrays/to-string";
 import { encode as varintEncode } from "varint";
 
-import { bytesToUtf8, utf8ToBytes } from "../utf8";
+import { bytesToHex, bytesToUtf8, hexToBytes, utf8ToBytes } from "../utils";
-import { bytesToHex, hexToBytes } from "../utils";
 
 import { ERR_INVALID_ID, ERR_NO_SIGNATURE, MAX_RECORD_SIZE } from "./constants";
 import {

src/lib/enr/keypair/secp256k1.ts

@@ -1,6 +1,7 @@
 import crypto from "crypto";
 
 import * as secp256k1 from "secp256k1";
+import { concat } from "uint8arrays/concat";
 
 import { AbstractKeypair, IKeypair, IKeypairClass, KeypairType } from "./types";
 
@@ -8,20 +9,14 @@ export function secp256k1PublicKeyToCompressed(
   publicKey: Uint8Array
 ): Uint8Array {
   if (publicKey.length === 64) {
-    const _publicKey = new Uint8Array(publicKey.length + 1);
+    publicKey = concat([[4], publicKey], 65);
-    _publicKey.set([4]);
-    _publicKey.set(publicKey, 1);
-    publicKey = _publicKey;
   }
   return secp256k1.publicKeyConvert(publicKey, true);
 }
 
 export function secp256k1PublicKeyToFull(publicKey: Uint8Array): Uint8Array {
   if (publicKey.length === 64) {
-    const _publicKey = new Uint8Array(publicKey.length + 1);
+    publicKey = concat([[4], publicKey], 65);
-    _publicKey.set([4]);
-    _publicKey.set(publicKey, 1);
-    publicKey = _publicKey;
   }
   return secp256k1.publicKeyConvert(publicKey, false);
 }
@@ -67,11 +62,7 @@ export const Secp256k1Keypair: IKeypairClass = class Secp256k1Keypair
 
   sign(msg: Uint8Array): Uint8Array {
     const { signature, recid } = secp256k1.ecdsaSign(msg, this.privateKey);
-
+    return concat([signature, [recid]], signature.length + 1);
-    const result = new Uint8Array(signature.length + 1);
-    result.set(signature);
-    result.set([recid], signature.length);
-    return result;
   }
 
   verify(msg: Uint8Array, sig: Uint8Array): boolean {

src/lib/utf8.ts

@@ -1,85 +0,0 @@
-/**
- * Decode bytes to utf-8 string.
- */
-// Thanks https://gist.github.com/pascaldekloe/62546103a1576803dade9269ccf76330
-export function bytesToUtf8(bytes: Uint8Array): string {
-  let i = 0,
-    s = "";
-  while (i < bytes.length) {
-    let c = bytes[i++];
-    if (c > 127) {
-      if (c > 191 && c < 224) {
-        if (i >= bytes.length)
-          throw new Error("UTF-8 decode: incomplete 2-byte sequence");
-        c = ((c & 31) << 6) | (bytes[i++] & 63);
-      } else if (c > 223 && c < 240) {
-        if (i + 1 >= bytes.length)
-          throw new Error("UTF-8 decode: incomplete 3-byte sequence");
-        c = ((c & 15) << 12) | ((bytes[i++] & 63) << 6) | (bytes[i++] & 63);
-      } else if (c > 239 && c < 248) {
-        if (i + 2 >= bytes.length)
-          throw new Error("UTF-8 decode: incomplete 4-byte sequence");
-        c =
-          ((c & 7) << 18) |
-          ((bytes[i++] & 63) << 12) |
-          ((bytes[i++] & 63) << 6) |
-          (bytes[i++] & 63);
-      } else
-        throw new Error(
-          "UTF-8 decode: unknown multi byte start 0x" +
-            c.toString(16) +
-            " at index " +
-            (i - 1)
-        );
-    }
-    if (c <= 0xffff) s += String.fromCharCode(c);
-    else if (c <= 0x10ffff) {
-      c -= 0x10000;
-      s += String.fromCharCode((c >> 10) | 0xd800);
-      s += String.fromCharCode((c & 0x3ff) | 0xdc00);
-    } else
-      throw new Error(
-        "UTF-8 decode: code point 0x" + c.toString(16) + " exceeds UTF-16 reach"
-      );
-  }
-  return s;
-}
-
-/**
- * Encode utf-8 string to byte array
- */
-// Thanks https://gist.github.com/pascaldekloe/62546103a1576803dade9269ccf76330
-export function utf8ToBytes(s: string): Uint8Array {
-  let i = 0;
-  const bytes = new Uint8Array(s.length * 4);
-  for (let ci = 0; ci != s.length; ci++) {
-    let c = s.charCodeAt(ci);
-    if (c < 128) {
-      bytes[i++] = c;
-      continue;
-    }
-    if (c < 2048) {
-      bytes[i++] = (c >> 6) | 192;
-    } else {
-      if (c > 0xd7ff && c < 0xdc00) {
-        if (++ci >= s.length)
-          throw new Error("UTF-8 encode: incomplete surrogate pair");
-        const c2 = s.charCodeAt(ci);
-        if (c2 < 0xdc00 || c2 > 0xdfff)
-          throw new Error(
-            "UTF-8 encode: second surrogate character 0x" +
-              c2.toString(16) +
-              " at index " +
-              ci +
-              " out of range"
-          );
-        c = 0x10000 + ((c & 0x03ff) << 10) + (c2 & 0x03ff);
-        bytes[i++] = (c >> 18) | 240;
-        bytes[i++] = ((c >> 12) & 63) | 128;
-      } else bytes[i++] = (c >> 12) | 224;
-      bytes[i++] = ((c >> 6) & 63) | 128;
-    }
-    bytes[i++] = (c & 63) | 128;
-  }
-  return bytes.subarray(0, i);
-}

src/lib/utils.ts

@@ -1,79 +1,25 @@
 import { keccak256, Message } from "js-sha3";
-
+import { fromString } from "uint8arrays/from-string";
-/**
+import { toString } from "uint8arrays/to-string";
- * Convert input to a Buffer.
- *
- * @deprecated Use `hexToBytes` instead.
- */
-export function hexToBuf(hex: string | Buffer | Uint8Array): Buffer {
-  if (typeof hex === "string") {
-    return Buffer.from(hex.replace(/^0x/i, ""), "hex");
-  } else {
-    return Buffer.from(hex);
-  }
-}
 
 /**
  * Convert input to a byte array.
+ *
+ * Handles both `0x` prefixed and non-prefixed strings.
  */
 export function hexToBytes(hex: string | Uint8Array): Uint8Array {
   if (typeof hex === "string") {
     const _hex = hex.replace(/^0x/i, "");
-    const bytes = [];
+    return fromString(_hex, "base16");
-    for (let c = 0; c < _hex.length; c += 2)
-      bytes.push(parseInt(_hex.substring(c, c + 2), 16));
-
-    return new Uint8Array(bytes);
   }
   return hex;
 }
 
-/**
- * Convert input to hex string (no `0x` prefix).
- *
- * @deprecated Use `bytesToHex` instead.
- */
-export function bufToHex(buf: Uint8Array | Buffer | ArrayBuffer): string {
-  const _buf = Buffer.from(buf);
-  return _buf.toString("hex");
-}
-
 /**
  * Convert byte array to hex string (no `0x` prefix).
  */
-export function bytesToHex(bytes: Uint8Array): string {
+export const bytesToHex = (bytes: Uint8Array): string =>
-  const hex = [];
+  toString(bytes, "base16");
-  for (let i = 0; i < bytes.length; i++) {
-    const current = bytes[i] < 0 ? bytes[i] + 256 : bytes[i];
-    hex.push((current >>> 4).toString(16));
-    hex.push((current & 0xf).toString(16));
-  }
-  return hex.join("");
-}
-
-/**
- * Compare both inputs, return true if they represent the same byte array.
- */
-export function equalByteArrays(
-  a: Uint8Array | string,
-  b: Uint8Array | string
-): boolean {
-  let _a: string;
-  let _b: string;
-  if (typeof a === "string") {
-    _a = a.replace(/^0x/i, "").toLowerCase();
-  } else {
-    _a = bytesToHex(a);
-  }
-
-  if (typeof b === "string") {
-    _b = b.replace(/^0x/i, "").toLowerCase();
-  } else {
-    _b = bytesToHex(b);
-  }
-
-  return _a === _b;
-}
 
 /**
  * Return Keccak-256 of the input.
@@ -81,3 +27,13 @@ export function equalByteArrays(
 export function keccak256Buf(message: Message): Uint8Array {
   return new Uint8Array(keccak256.arrayBuffer(message));
 }
+
+/**
+ * Decode byte array to utf-8 string.
+ */
+export const bytesToUtf8 = (b: Uint8Array): string => toString(b, "utf8");
+
+/**
+ * Encode utf-8 string to byte array.
+ */
+export const utf8ToBytes = (s: string): Uint8Array => fromString(s, "utf8");

src/lib/waku_message/ecies.ts

@@ -1,8 +1,8 @@
 import * as secp from "@noble/secp256k1";
+import { concat } from "uint8arrays/concat";
 
 import { randomBytes, sha256, subtle } from "../crypto";
 import { hexToBytes } from "../utils";
-
 /**
  * HKDF as implemented in go-ethereum.
  */
@@ -12,17 +12,18 @@ function kdf(secret: Uint8Array, outputLength: number): Promise<Uint8Array> {
   let willBeResult = Promise.resolve(new Uint8Array());
   while (written < outputLength) {
     const counters = new Uint8Array([ctr >> 24, ctr >> 16, ctr >> 8, ctr]);
-    const countersSecret = new Uint8Array(counters.length + secret.length);
+    const countersSecret = concat(
-    countersSecret.set(counters, 0);
+      [counters, secret],
-    countersSecret.set(secret, counters.length);
+      counters.length + secret.length
+    );
     const willBeHashResult = sha256(countersSecret);
     willBeResult = willBeResult.then((result) =>
       willBeHashResult.then((hashResult) => {
         const _hashResult = new Uint8Array(hashResult);
-        const _res = new Uint8Array(result.length + _hashResult.length);
+        return concat(
-        _res.set(result, 0);
+          [result, _hashResult],
-        _res.set(_hashResult, result.length);
+          result.length + _hashResult.length
-        return _res;
+        );
       })
     );
     written += 32;
@@ -135,24 +136,16 @@ export async function encrypt(
   const encryptionKey = hash.slice(0, 16);
   const cipherText = await aesCtrEncrypt(iv, encryptionKey, msg);
 
-  const ivCipherText = new Uint8Array(iv.length + cipherText.length);
+  const ivCipherText = concat([iv, cipherText], iv.length + cipherText.length);
-  ivCipherText.set(iv, 0);
-  ivCipherText.set(cipherText, iv.length);
 
   const macKey = await sha256(hash.slice(16));
   const hmac = await hmacSha256Sign(macKey, ivCipherText);
   const ephemPublicKey = secp.getPublicKey(ephemPrivateKey, false);
 
-  const cipher = new Uint8Array(
+  return concat(
+    [ephemPublicKey, ivCipherText, hmac],
     ephemPublicKey.length + ivCipherText.length + hmac.length
   );
-  let index = 0;
-  cipher.set(ephemPublicKey, index);
-  index += ephemPublicKey.length;
-  cipher.set(ivCipherText, index);
-  index += ivCipherText.length;
-  cipher.set(hmac, index);
-  return cipher;
 }
 
 const metaLength = 1 + 64 + 16 + 32;

src/lib/waku_message/index.node.spec.ts

@@ -8,8 +8,7 @@ import {
   WakuRelayMessage,
 } from "../../test_utils";
 import { delay } from "../../test_utils/delay";
-import { bytesToUtf8, utf8ToBytes } from "../utf8";
+import { bytesToHex, bytesToUtf8, hexToBytes, utf8ToBytes } from "../utils";
-import { bytesToHex, hexToBytes } from "../utils";
 import { Protocols, Waku } from "../waku";
 
 import {

src/lib/waku_message/index.ts

@@ -3,7 +3,7 @@ import Long from "long";
 import { Reader } from "protobufjs/minimal";
 
 import * as proto from "../../proto/waku/v2/message";
-import { bytesToUtf8, utf8ToBytes } from "../utf8";
+import { bytesToUtf8, utf8ToBytes } from "../utils";
 
 import * as version_1 from "./version_1";
 
@@ -252,12 +252,12 @@ export class WakuMessage {
   }
 
   get payloadAsUtf8(): string {
-    if (!this.proto.payload) {
+    if (!this.payload) {
       return "";
     }
 
     try {
-      return bytesToUtf8(this.proto.payload);
+      return bytesToUtf8(this.payload);
     } catch (e) {
       dbg("Could not decode byte as UTF-8", e);
       return "";
@@ -265,7 +265,10 @@ export class WakuMessage {
   }
 
   get payload(): Uint8Array | undefined {
-    return this.proto.payload;
+    if (this.proto.payload) {
+      return new Uint8Array(this.proto.payload);
+    }
+    return;
   }
 
   get contentTopic(): string | undefined {