634: Replace conversion functions with `uint8arrays` r=D4nte a=D4nte

Resolves #603

Co-authored-by: Franck Royer <franck@status.im>
This commit is contained in:
status-bors-ng[bot] 2022-03-25 06:08:53 +00:00 committed by GitHub
commit 0784cf2545
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 69 additions and 204 deletions

View File

@ -10,10 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Examples: Add Relay JavaScript example.
- **Breaking**: Moved utf-8 conversion functions to `utils`.
### Fixed
- Replace Base 64 buggy conversion functions with `uint8arrays`.
- Replace Base 64 buggy conversion functions with `uint8arrays`.
### Removed
- **Breaking**: Removed `equalByteArrays`, use `uint8arrays/equals` instead.
See changes in `eth-pm` example.
- **Breaking**: Removed deprecated utils functions.
## [0.19.2] - 2022-03-21

View File

@ -1,8 +1,9 @@
import "@ethersproject/shims";
import { PublicKeyMessage } from "./messaging/wire";
import { hexToBytes, equalByteArrays, bytesToHex } from "js-waku/lib/utils";
import { hexToBytes, bytesToHex } from "js-waku/lib/utils";
import * as sigUtil from "eth-sig-util";
import { equals } from "uint8arrays/equals";
/**
* Sign the encryption public key with Web3. This can then be published to let other
@ -97,5 +98,5 @@ export function validatePublicKeyMessage(msg: PublicKeyMessage): boolean {
console.log("Recovered", recovered);
console.log("ethAddress", "0x" + bytesToHex(msg.ethAddress));
return equalByteArrays(recovered, msg.ethAddress);
return equals(hexToBytes(recovered), msg.ethAddress);
}

View File

@ -3,7 +3,8 @@ import { Waku, WakuMessage } from "js-waku";
import { PrivateMessage, PublicKeyMessage } from "./messaging/wire";
import { validatePublicKeyMessage } from "./crypto";
import { Message } from "./messaging/Messages";
import { bytesToHex, equalByteArrays } from "js-waku/lib/utils";
import { bytesToHex, hexToBytes } from "js-waku/lib/utils";
import { equals } from "uint8arrays/equals";
export const PublicKeyContentTopic =
"/eth-pm-wallet/1/encryption-public-key/proto";
@ -36,7 +37,8 @@ export function handlePublicKeyMessage(
if (!msg.payload) return;
const publicKeyMsg = PublicKeyMessage.decode(msg.payload);
if (!publicKeyMsg) return;
if (myAddress && equalByteArrays(publicKeyMsg.ethAddress, myAddress)) return;
if (myAddress && equals(publicKeyMsg.ethAddress, hexToBytes(myAddress)))
return;
const res = validatePublicKeyMessage(publicKeyMsg);
console.log("Is Public Key Message valid?", res);
@ -77,7 +79,7 @@ export async function handlePrivateMessage(
console.log("Failed to decode Private Message");
return;
}
if (!equalByteArrays(privateMessage.toAddress, address)) return;
if (!equals(privateMessage.toAddress, hexToBytes(address))) return;
const timestamp = wakuMsg.timestamp ? wakuMsg.timestamp : new Date();

View File

@ -1,11 +1,12 @@
import "@ethersproject/shims";
import { PublicKeyMessage } from "./messaging/wire";
import { hexToBytes, equalByteArrays, bytesToHex } from "js-waku/lib/utils";
import { hexToBytes, bytesToHex } from "js-waku/lib/utils";
import { generatePrivateKey, getPublicKey } from "js-waku";
import * as sigUtil from "eth-sig-util";
import { PublicKeyContentTopic } from "./waku";
import { keccak256 } from "ethers/lib/utils";
import { equals } from "uint8arrays/equals";
export const PublicKeyMessageEncryptionKey = hexToBytes(
keccak256(Buffer.from(PublicKeyContentTopic, "utf-8"))
@ -118,5 +119,5 @@ export function validatePublicKeyMessage(msg: PublicKeyMessage): boolean {
console.log("Recovered", recovered);
console.log("ethAddress", "0x" + bytesToHex(msg.ethAddress));
return equalByteArrays(recovered, msg.ethAddress);
return equals(hexToBytes(recovered), msg.ethAddress);
}

View File

@ -3,7 +3,8 @@ import { Waku, WakuMessage } from "js-waku";
import { PrivateMessage, PublicKeyMessage } from "./messaging/wire";
import { validatePublicKeyMessage } from "./crypto";
import { Message } from "./messaging/Messages";
import { bytesToHex, equalByteArrays } from "js-waku/lib/utils";
import { bytesToHex, hexToBytes } from "js-waku/lib/utils";
import { equals } from "uint8arrays/equals";
export const PublicKeyContentTopic = "/eth-pm/1/public-key/proto";
export const PrivateMessageContentTopic = "/eth-pm/1/private-message/proto";
@ -34,7 +35,8 @@ export function handlePublicKeyMessage(
if (!msg.payload) return;
const publicKeyMsg = PublicKeyMessage.decode(msg.payload);
if (!publicKeyMsg) return;
if (myAddress && equalByteArrays(publicKeyMsg.ethAddress, myAddress)) return;
if (myAddress && equals(publicKeyMsg.ethAddress, hexToBytes(myAddress)))
return;
const res = validatePublicKeyMessage(publicKeyMsg);
console.log("Is Public Key Message valid?", res);
@ -62,7 +64,7 @@ export async function handlePrivateMessage(
console.log("Failed to decode Private Message");
return;
}
if (!equalByteArrays(privateMessage.toAddress, address)) return;
if (!equals(privateMessage.toAddress, hexToBytes(address))) return;
const timestamp = wakuMsg.timestamp ? wakuMsg.timestamp : new Date();

View File

@ -3,8 +3,6 @@ export * as discovery from "./lib/discovery";
export * as enr from "./lib/enr";
export * as utf8 from "./lib/utf8";
export * as utils from "./lib/utils";
export * as waku from "./lib/waku";

View File

@ -6,7 +6,7 @@ import {
query,
} from "dns-query";
import { bytesToUtf8 } from "../utf8";
import { bytesToUtf8 } from "../utils";
import { DnsClient } from "./dns";

View File

@ -5,8 +5,7 @@ import { ecdsaVerify } from "secp256k1";
import { fromString } from "uint8arrays/from-string";
import { ENR } from "../enr";
import { utf8ToBytes } from "../utf8";
import { keccak256Buf } from "../utils";
import { keccak256Buf, utf8ToBytes } from "../utils";
export type ENRRootValues = {
eRoot: string;

View File

@ -2,8 +2,7 @@ import { assert, expect } from "chai";
import { Multiaddr } from "multiaddr";
import PeerId from "peer-id";
import { utf8ToBytes } from "../utf8";
import { bytesToHex, hexToBytes } from "../utils";
import { bytesToHex, hexToBytes, utf8ToBytes } from "../utils";
import { ERR_INVALID_ID } from "./constants";
import { ENR } from "./enr";

View File

@ -9,8 +9,7 @@ import { fromString } from "uint8arrays/from-string";
import { toString } from "uint8arrays/to-string";
import { encode as varintEncode } from "varint";
import { bytesToUtf8, utf8ToBytes } from "../utf8";
import { bytesToHex, hexToBytes } from "../utils";
import { bytesToHex, bytesToUtf8, hexToBytes, utf8ToBytes } from "../utils";
import { ERR_INVALID_ID, ERR_NO_SIGNATURE, MAX_RECORD_SIZE } from "./constants";
import {

View File

@ -1,6 +1,7 @@
import crypto from "crypto";
import * as secp256k1 from "secp256k1";
import { concat } from "uint8arrays/concat";
import { AbstractKeypair, IKeypair, IKeypairClass, KeypairType } from "./types";
@ -8,20 +9,14 @@ export function secp256k1PublicKeyToCompressed(
publicKey: Uint8Array
): Uint8Array {
if (publicKey.length === 64) {
const _publicKey = new Uint8Array(publicKey.length + 1);
_publicKey.set([4]);
_publicKey.set(publicKey, 1);
publicKey = _publicKey;
publicKey = concat([[4], publicKey], 65);
}
return secp256k1.publicKeyConvert(publicKey, true);
}
export function secp256k1PublicKeyToFull(publicKey: Uint8Array): Uint8Array {
if (publicKey.length === 64) {
const _publicKey = new Uint8Array(publicKey.length + 1);
_publicKey.set([4]);
_publicKey.set(publicKey, 1);
publicKey = _publicKey;
publicKey = concat([[4], publicKey], 65);
}
return secp256k1.publicKeyConvert(publicKey, false);
}
@ -67,11 +62,7 @@ export const Secp256k1Keypair: IKeypairClass = class Secp256k1Keypair
sign(msg: Uint8Array): Uint8Array {
const { signature, recid } = secp256k1.ecdsaSign(msg, this.privateKey);
const result = new Uint8Array(signature.length + 1);
result.set(signature);
result.set([recid], signature.length);
return result;
return concat([signature, [recid]], signature.length + 1);
}
verify(msg: Uint8Array, sig: Uint8Array): boolean {

View File

@ -1,85 +0,0 @@
/**
* Decode bytes to utf-8 string.
*/
// Thanks https://gist.github.com/pascaldekloe/62546103a1576803dade9269ccf76330
export function bytesToUtf8(bytes: Uint8Array): string {
let i = 0,
s = "";
while (i < bytes.length) {
let c = bytes[i++];
if (c > 127) {
if (c > 191 && c < 224) {
if (i >= bytes.length)
throw new Error("UTF-8 decode: incomplete 2-byte sequence");
c = ((c & 31) << 6) | (bytes[i++] & 63);
} else if (c > 223 && c < 240) {
if (i + 1 >= bytes.length)
throw new Error("UTF-8 decode: incomplete 3-byte sequence");
c = ((c & 15) << 12) | ((bytes[i++] & 63) << 6) | (bytes[i++] & 63);
} else if (c > 239 && c < 248) {
if (i + 2 >= bytes.length)
throw new Error("UTF-8 decode: incomplete 4-byte sequence");
c =
((c & 7) << 18) |
((bytes[i++] & 63) << 12) |
((bytes[i++] & 63) << 6) |
(bytes[i++] & 63);
} else
throw new Error(
"UTF-8 decode: unknown multi byte start 0x" +
c.toString(16) +
" at index " +
(i - 1)
);
}
if (c <= 0xffff) s += String.fromCharCode(c);
else if (c <= 0x10ffff) {
c -= 0x10000;
s += String.fromCharCode((c >> 10) | 0xd800);
s += String.fromCharCode((c & 0x3ff) | 0xdc00);
} else
throw new Error(
"UTF-8 decode: code point 0x" + c.toString(16) + " exceeds UTF-16 reach"
);
}
return s;
}
/**
* Encode utf-8 string to byte array
*/
// Thanks https://gist.github.com/pascaldekloe/62546103a1576803dade9269ccf76330
export function utf8ToBytes(s: string): Uint8Array {
let i = 0;
const bytes = new Uint8Array(s.length * 4);
for (let ci = 0; ci != s.length; ci++) {
let c = s.charCodeAt(ci);
if (c < 128) {
bytes[i++] = c;
continue;
}
if (c < 2048) {
bytes[i++] = (c >> 6) | 192;
} else {
if (c > 0xd7ff && c < 0xdc00) {
if (++ci >= s.length)
throw new Error("UTF-8 encode: incomplete surrogate pair");
const c2 = s.charCodeAt(ci);
if (c2 < 0xdc00 || c2 > 0xdfff)
throw new Error(
"UTF-8 encode: second surrogate character 0x" +
c2.toString(16) +
" at index " +
ci +
" out of range"
);
c = 0x10000 + ((c & 0x03ff) << 10) + (c2 & 0x03ff);
bytes[i++] = (c >> 18) | 240;
bytes[i++] = ((c >> 12) & 63) | 128;
} else bytes[i++] = (c >> 12) | 224;
bytes[i++] = ((c >> 6) & 63) | 128;
}
bytes[i++] = (c & 63) | 128;
}
return bytes.subarray(0, i);
}

View File

@ -1,79 +1,25 @@
import { keccak256, Message } from "js-sha3";
/**
* Convert input to a Buffer.
*
* @deprecated Use `hexToBytes` instead.
*/
export function hexToBuf(hex: string | Buffer | Uint8Array): Buffer {
if (typeof hex === "string") {
return Buffer.from(hex.replace(/^0x/i, ""), "hex");
} else {
return Buffer.from(hex);
}
}
import { fromString } from "uint8arrays/from-string";
import { toString } from "uint8arrays/to-string";
/**
* Convert input to a byte array.
*
* Handles both `0x` prefixed and non-prefixed strings.
*/
export function hexToBytes(hex: string | Uint8Array): Uint8Array {
if (typeof hex === "string") {
const _hex = hex.replace(/^0x/i, "");
const bytes = [];
for (let c = 0; c < _hex.length; c += 2)
bytes.push(parseInt(_hex.substring(c, c + 2), 16));
return new Uint8Array(bytes);
return fromString(_hex, "base16");
}
return hex;
}
/**
* Convert input to hex string (no `0x` prefix).
*
* @deprecated Use `bytesToHex` instead.
*/
export function bufToHex(buf: Uint8Array | Buffer | ArrayBuffer): string {
const _buf = Buffer.from(buf);
return _buf.toString("hex");
}
/**
* Convert byte array to hex string (no `0x` prefix).
*/
export function bytesToHex(bytes: Uint8Array): string {
const hex = [];
for (let i = 0; i < bytes.length; i++) {
const current = bytes[i] < 0 ? bytes[i] + 256 : bytes[i];
hex.push((current >>> 4).toString(16));
hex.push((current & 0xf).toString(16));
}
return hex.join("");
}
/**
* Compare both inputs, return true if they represent the same byte array.
*/
export function equalByteArrays(
a: Uint8Array | string,
b: Uint8Array | string
): boolean {
let _a: string;
let _b: string;
if (typeof a === "string") {
_a = a.replace(/^0x/i, "").toLowerCase();
} else {
_a = bytesToHex(a);
}
if (typeof b === "string") {
_b = b.replace(/^0x/i, "").toLowerCase();
} else {
_b = bytesToHex(b);
}
return _a === _b;
}
export const bytesToHex = (bytes: Uint8Array): string =>
toString(bytes, "base16");
/**
* Return Keccak-256 of the input.
@ -81,3 +27,13 @@ export function equalByteArrays(
export function keccak256Buf(message: Message): Uint8Array {
return new Uint8Array(keccak256.arrayBuffer(message));
}
/**
* Decode byte array to utf-8 string.
*/
export const bytesToUtf8 = (b: Uint8Array): string => toString(b, "utf8");
/**
* Encode utf-8 string to byte array.
*/
export const utf8ToBytes = (s: string): Uint8Array => fromString(s, "utf8");

View File

@ -1,8 +1,8 @@
import * as secp from "@noble/secp256k1";
import { concat } from "uint8arrays/concat";
import { randomBytes, sha256, subtle } from "../crypto";
import { hexToBytes } from "../utils";
/**
* HKDF as implemented in go-ethereum.
*/
@ -12,17 +12,18 @@ function kdf(secret: Uint8Array, outputLength: number): Promise<Uint8Array> {
let willBeResult = Promise.resolve(new Uint8Array());
while (written < outputLength) {
const counters = new Uint8Array([ctr >> 24, ctr >> 16, ctr >> 8, ctr]);
const countersSecret = new Uint8Array(counters.length + secret.length);
countersSecret.set(counters, 0);
countersSecret.set(secret, counters.length);
const countersSecret = concat(
[counters, secret],
counters.length + secret.length
);
const willBeHashResult = sha256(countersSecret);
willBeResult = willBeResult.then((result) =>
willBeHashResult.then((hashResult) => {
const _hashResult = new Uint8Array(hashResult);
const _res = new Uint8Array(result.length + _hashResult.length);
_res.set(result, 0);
_res.set(_hashResult, result.length);
return _res;
return concat(
[result, _hashResult],
result.length + _hashResult.length
);
})
);
written += 32;
@ -135,24 +136,16 @@ export async function encrypt(
const encryptionKey = hash.slice(0, 16);
const cipherText = await aesCtrEncrypt(iv, encryptionKey, msg);
const ivCipherText = new Uint8Array(iv.length + cipherText.length);
ivCipherText.set(iv, 0);
ivCipherText.set(cipherText, iv.length);
const ivCipherText = concat([iv, cipherText], iv.length + cipherText.length);
const macKey = await sha256(hash.slice(16));
const hmac = await hmacSha256Sign(macKey, ivCipherText);
const ephemPublicKey = secp.getPublicKey(ephemPrivateKey, false);
const cipher = new Uint8Array(
return concat(
[ephemPublicKey, ivCipherText, hmac],
ephemPublicKey.length + ivCipherText.length + hmac.length
);
let index = 0;
cipher.set(ephemPublicKey, index);
index += ephemPublicKey.length;
cipher.set(ivCipherText, index);
index += ivCipherText.length;
cipher.set(hmac, index);
return cipher;
}
const metaLength = 1 + 64 + 16 + 32;

View File

@ -8,8 +8,7 @@ import {
WakuRelayMessage,
} from "../../test_utils";
import { delay } from "../../test_utils/delay";
import { bytesToUtf8, utf8ToBytes } from "../utf8";
import { bytesToHex, hexToBytes } from "../utils";
import { bytesToHex, bytesToUtf8, hexToBytes, utf8ToBytes } from "../utils";
import { Protocols, Waku } from "../waku";
import {

View File

@ -3,7 +3,7 @@ import Long from "long";
import { Reader } from "protobufjs/minimal";
import * as proto from "../../proto/waku/v2/message";
import { bytesToUtf8, utf8ToBytes } from "../utf8";
import { bytesToUtf8, utf8ToBytes } from "../utils";
import * as version_1 from "./version_1";
@ -252,12 +252,12 @@ export class WakuMessage {
}
get payloadAsUtf8(): string {
if (!this.proto.payload) {
if (!this.payload) {
return "";
}
try {
return bytesToUtf8(this.proto.payload);
return bytesToUtf8(this.payload);
} catch (e) {
dbg("Could not decode byte as UTF-8", e);
return "";
@ -265,7 +265,10 @@ export class WakuMessage {
}
get payload(): Uint8Array | undefined {
return this.proto.payload;
if (this.proto.payload) {
return new Uint8Array(this.proto.payload);
}
return;
}
get contentTopic(): string | undefined {