Added more general error handling (e.g. error, ignore, replace) for calling toUtf8String.

packages/strings/src.ts/index.ts

@@ -2,7 +2,7 @@
 
 import { formatBytes32String, parseBytes32String } from "./bytes32";
 import { nameprep } from "./idna";
-import { _toEscapedUtf8String, toUtf8Bytes, toUtf8CodePoints, toUtf8String, UnicodeNormalizationForm } from "./utf8";
+import { _toEscapedUtf8String, toUtf8Bytes, toUtf8CodePoints, toUtf8String, UnicodeNormalizationForm, Utf8ErrorFunc, Utf8ErrorFuncs, Utf8ErrorReason } from "./utf8";
 
 export {
     _toEscapedUtf8String,
@@ -10,6 +10,10 @@ export {
     toUtf8CodePoints,
     toUtf8String,
 
+    Utf8ErrorFunc,
+    Utf8ErrorFuncs,
+    Utf8ErrorReason,
+
     UnicodeNormalizationForm,
 
     formatBytes32String,

packages/strings/src.ts/utf8.ts

@@ -16,8 +16,94 @@ export enum UnicodeNormalizationForm {
     NFKD     = "NFKD"
 };
 
+export enum Utf8ErrorReason {
+    // A continuation byte was present where there was nothing to continue
+    // - offset = the index the codepoint began in
+    UNEXPECTED_CONTINUE   = "unexpected continuation byte",
+
+    // An invalid (non-continuation) byte to start a UTF-8 codepoint was found
+    // - offset = the index the codepoint began in
+    BAD_PREFIX            = "bad codepoint prefix",
+
+    // The string is too short to process the expected codepoint
+    // - offset = the index the codepoint began in
+    OVERRUN               = "string overrun",
+
+    // A missing continuation byte was expected but not found
+    // - offset = the index the continuation byte was expected at
+    MISSING_CONTINUE      = "missing continuation byte",
+
+    // The computed code point is outside the range for UTF-8
+    // - offset       = start of this codepoint
+    // - badCodepoint = the computed codepoint; outside the UTF-8 range
+    OUT_OF_RANGE          = "out of UTF-8 range",
+
+    // UTF-8 strings may not contain UTF-16 surrogate pairs
+    // - offset       = start of this codepoint
+    // - badCodepoint = the computed codepoint; inside the UTF-16 surrogate range
+    UTF16_SURROGATE       = "UTF-16 surrogate",
+
+    // The string is an overlong reperesentation
+    // - offset       = start of this codepoint
+    // - badCodepoint = the computed codepoint; already bounds checked
+    OVERLONG              = "overlong representation",
+};
+
+
+export type Utf8ErrorFunc = (reason: Utf8ErrorReason, offset: number, bytes: ArrayLike<number>, output: Array<number>, badCodepoint?: number) => number;
+
+function errorFunc(reason: Utf8ErrorReason, offset: number, bytes: ArrayLike<number>, output: Array<number>, badCodepoint?: number): number {
+    return logger.throwArgumentError(`invalid codepoint at offset ${ offset }; ${ reason }`, "bytes", bytes);
+}
+
+function ignoreFunc(reason: Utf8ErrorReason, offset: number, bytes: ArrayLike<number>, output: Array<number>, badCodepoint?: number): number {
+
+    // If there is an invalid prefix (including stray continuation), skip any additional continuation bytes
+    if (reason === Utf8ErrorReason.BAD_PREFIX || reason === Utf8ErrorReason.UNEXPECTED_CONTINUE) {
+        let i = 0;
+        for (let o = offset + 1; o < bytes.length; o++) {
+            if (bytes[o] >> 6 !== 0x02) { break; }
+            i++;
+        }
+        return i;
+    }
+
+    // This byte runs us past the end of the string, so just jump to the end
+    // (but the first byte was read already read and therefore skipped)
+    if (reason === Utf8ErrorReason.OVERRUN) {
+        return bytes.length - offset - 1;
+    }
+
+    // Nothing to skip
+    return 0;
+}
+
+function replaceFunc(reason: Utf8ErrorReason, offset: number, bytes: ArrayLike<number>, output: Array<number>, badCodepoint?: number): number {
+
+    // Overlong representations are otherwise "valid" code points; just non-deistingtished
+    if (reason === Utf8ErrorReason.OVERLONG) {
+        output.push(badCodepoint);
+        return 0;
+    }
+
+    // Put the replacement character into the output
+    output.push(0xfffd);
+
+    // Otherwise, process as if ignoring errors
+    return ignoreFunc(reason, offset, bytes, output, badCodepoint);
+}
+
+// Common error handing strategies
+export const Utf8ErrorFuncs: { [ name: string ]: Utf8ErrorFunc } = Object.freeze({
+    error: errorFunc,
+    ignore: ignoreFunc,
+    replace: replaceFunc
+});
+
 // http://stackoverflow.com/questions/13356493/decode-utf-8-with-javascript#13691499
-function getUtf8CodePoints(bytes: BytesLike, ignoreErrors?: boolean): Array<number> {
+function getUtf8CodePoints(bytes: BytesLike, onError?: Utf8ErrorFunc): Array<number> {
+    if (onError == null) { onError = Utf8ErrorFuncs.error; }
+
     bytes = arrayify(bytes);
 
     const result: Array<number> = [];
@@ -27,6 +113,7 @@ function getUtf8CodePoints(bytes: BytesLike, ignoreErrors?: boolean): Array<numb
     while(i < bytes.length) {
 
         const c = bytes[i++];
+
         // 0xxx xxxx
         if (c >> 7 === 0) {
             result.push(c);
@@ -53,24 +140,17 @@ function getUtf8CodePoints(bytes: BytesLike, ignoreErrors?: boolean): Array<numb
             overlongMask = 0xffff;
 
         } else {
-            if (!ignoreErrors) {
+            if ((c & 0xc0) === 0x80) {
-                if ((c & 0xc0) === 0x80) {
+                i += onError(Utf8ErrorReason.UNEXPECTED_CONTINUE, i - 1, bytes, result);
-                    throw new Error("invalid utf8 byte sequence; unexpected continuation byte");
+            } else {
-                }
+                i += onError(Utf8ErrorReason.BAD_PREFIX, i - 1, bytes, result);
-                throw new Error("invalid utf8 byte sequence; invalid prefix");
             }
             continue;
         }
 
         // Do we have enough bytes in our data?
-        if (i + extraLength > bytes.length) {
+        if (i - 1 + extraLength >= bytes.length) {
-            if (!ignoreErrors) { throw new Error("invalid utf8 byte sequence; too short"); }
+            i += onError(Utf8ErrorReason.OVERRUN, i - 1, bytes, result);
-
-            // If there is an invalid unprocessed byte, skip continuation bytes
-            for (; i < bytes.length; i++) {
-                if (bytes[i] >> 6 !== 0x02) { break; }
-            }
-
             continue;
         }
 
@@ -82,6 +162,7 @@ function getUtf8CodePoints(bytes: BytesLike, ignoreErrors?: boolean): Array<numb
 
             // Invalid continuation byte
             if ((nextChar & 0xc0) != 0x80) {
+                i += onError(Utf8ErrorReason.MISSING_CONTINUE, i, bytes, result);
                 res = null;
                 break;
             };
@@ -90,26 +171,24 @@ function getUtf8CodePoints(bytes: BytesLike, ignoreErrors?: boolean): Array<numb
             i++;
         }
 
-        if (res === null) {
+        // See above loop for invalid contimuation byte
-            if (!ignoreErrors) { throw new Error("invalid utf8 byte sequence; invalid continuation byte"); }
+        if (res === null) { continue; }
-            continue;
-        }
-
-        // Check for overlong seuences (more bytes than needed)
-        if (res <= overlongMask) {
-            if (!ignoreErrors) { throw new Error("invalid utf8 byte sequence; overlong"); }
-            continue;
-        }
 
         // Maximum code point
         if (res > 0x10ffff) {
-            if (!ignoreErrors) { throw new Error("invalid utf8 byte sequence; out-of-range"); }
+            i += onError(Utf8ErrorReason.OUT_OF_RANGE, i - 1 - extraLength, bytes, result, res);
             continue;
         }
 
         // Reserved for UTF-16 surrogate halves
         if (res >= 0xd800 && res <= 0xdfff) {
-            if (!ignoreErrors) { throw new Error("invalid utf8 byte sequence; utf-16 surrogate"); }
+            i += onError(Utf8ErrorReason.UTF16_SURROGATE, i - 1 - extraLength, bytes, result, res);
+            continue;
+        }
+
+        // Check for overlong sequences (more bytes than needed)
+        if (res <= overlongMask) {
+            i += onError(Utf8ErrorReason.OVERLONG, i - 1 - extraLength, bytes, result, res);
             continue;
         }
 
@@ -168,8 +247,8 @@ function escapeChar(value: number) {
     return "\\u" + hex.substring(hex.length - 4);
 }
 
-export function _toEscapedUtf8String(bytes: BytesLike, ignoreErrors?: boolean): string {
+export function _toEscapedUtf8String(bytes: BytesLike, onError?: Utf8ErrorFunc): string {
-    return '"' + getUtf8CodePoints(bytes, ignoreErrors).map((codePoint) => {
+    return '"' + getUtf8CodePoints(bytes, onError).map((codePoint) => {
         if (codePoint < 256) {
             switch (codePoint) {
                 case 8:  return "\\b";
@@ -207,8 +286,8 @@ export function _toUtf8String(codePoints: Array<number>): string {
     }).join("");
 }
 
-export function toUtf8String(bytes: BytesLike, ignoreErrors?: boolean): string {
+export function toUtf8String(bytes: BytesLike, onError?: Utf8ErrorFunc): string {
-    return _toUtf8String(getUtf8CodePoints(bytes, ignoreErrors));
+    return _toUtf8String(getUtf8CodePoints(bytes, onError));
 }
 
 export function toUtf8CodePoints(str: string, form: UnicodeNormalizationForm = UnicodeNormalizationForm.current): Array<number> {

packages/tests/src.ts/test-utils.ts

@@ -333,39 +333,55 @@ describe('Test Base64 coder', function() {
 });
 
 describe('Test UTF-8 coder', function() {
+    const overlong = ethers.utils.Utf8ErrorReason.OVERLONG;
+    const utf16Surrogate = ethers.utils.Utf8ErrorReason.UTF16_SURROGATE;
+    const overrun = ethers.utils.Utf8ErrorReason.OVERRUN;
+    const missingContinue = ethers.utils.Utf8ErrorReason.MISSING_CONTINUE;
+    const unexpectedContinue = ethers.utils.Utf8ErrorReason.UNEXPECTED_CONTINUE;
+    const outOfRange = ethers.utils.Utf8ErrorReason.OUT_OF_RANGE;
+
     let BadUTF = [
         // See: https://en.wikipedia.org/wiki/UTF-8#Overlong_encodings
-        { bytes: [ 0xF0,0x82, 0x82, 0xAC ], reason: 'overlong', name: 'wikipedia overlong encoded Euro sign' },
+        { bytes: [ 0xF0,0x82, 0x82, 0xAC ], reason: overlong, ignored: "", replaced: "\u20ac", name: 'wikipedia overlong encoded Euro sign' },
-        { bytes: [ 0xc0, 0x80 ], reason: 'overlong', name: '2-byte overlong - 0xc080' },
+        { bytes: [ 0xc0, 0x80 ], reason: overlong, ignored: "", replaced: "\u0000", name: '2-byte overlong - 0xc080' },
-        { bytes: [ 0xc0, 0xbf ], reason: 'overlong', name: '2-byte overlong - 0xc0bf' },
+        { bytes: [ 0xc0, 0xbf ], reason: overlong, ignored: "", replaced: "?", name: '2-byte overlong - 0xc0bf' },
-        { bytes: [ 0xc1, 0x80 ], reason: 'overlong', name: '2-byte overlong - 0xc180' },
+        { bytes: [ 0xc1, 0x80 ], reason: overlong, ignored: "", replaced: "@", name: '2-byte overlong - 0xc180' },
-        { bytes: [ 0xc1, 0xbf ], reason: 'overlong', name: '2-byte overlong - 0xc1bf' },
+        { bytes: [ 0xc1, 0xbf ], reason: overlong, ignored: "", replaced: "\u007f", name: '2-byte overlong - 0xc1bf' },
 
         // Reserved UTF-16 Surrogate halves
-        { bytes: [ 0xed, 0xa0, 0x80 ], reason: 'utf-16 surrogate', name: 'utf-16 surrogate - U+d800' },
+        { bytes: [ 0xed, 0xa0, 0x80 ], reason: utf16Surrogate, ignored: "", replaced: "\ufffd", name: 'utf-16 surrogate - U+d800' },
-        { bytes: [ 0xed, 0xbf, 0xbf ], reason: 'utf-16 surrogate', name: 'utf-16 surrogate - U+dfff' },
+        { bytes: [ 0xed, 0xbf, 0xbf ], reason: utf16Surrogate, ignored: "", replaced: "\ufffd", name: 'utf-16 surrogate - U+dfff' },
 
         // a leading byte not followed by enough continuation bytes
-        { bytes: [ 0xdf ], reason: 'too short', name: 'too short - 2-bytes - 0x00' },
+        { bytes: [ 0xdf ], reason: overrun, ignored: "", replaced: "\ufffd", name: 'too short - 2-bytes - 0x00' },
-        { bytes: [ 0xe0 ], reason: 'too short', name: 'too short - 3-bytes' },
+        { bytes: [ 0xe0 ], reason: overrun, ignored: "", replaced: "\ufffd", name: 'too short - 3-bytes' },
-        { bytes: [ 0xe0, 0x80 ], reason: 'too short', name: 'too short - 3-bytes with 1' },
+        { bytes: [ 0xe0, 0x80 ], reason: overrun, ignored: "", replaced: "\ufffd", name: 'too short - 3-bytes with 1' },
 
-        { bytes: [ 0x80 ], reason: 'unexpected continuation byte', name: 'unexpected continuation byte' },
+        { bytes: [ 0x80 ], reason: unexpectedContinue, ignored: "", replaced: "\ufffd", name: 'unexpected continuation byte' },
-        { bytes: [ 0xc2, 0x00 ], reason: 'invalid continuation byte', name: 'invalid continuation byte - 0xc200' },
+        { bytes: [ 0xc2, 0x00 ], reason: missingContinue, ignored: "\u0000", replaced: "\ufffd\u0000", name: 'invalid continuation byte - 0xc200' },
-        { bytes: [ 0xc2, 0x40 ], reason: 'invalid continuation byte', name: 'invalid continuation byte - 0xc240' },
+        { bytes: [ 0xc2, 0x40 ], reason: missingContinue, ignored: "@", replaced: "\ufffd@", name: 'invalid continuation byte - 0xc240' },
-        { bytes: [ 0xc2, 0xc0 ], reason: 'invalid continuation byte', name: 'invalid continuation byte - 0xc2c0' },
+        { bytes: [ 0xc2, 0xc0 ], reason: missingContinue, ignored: "", replaced: "\ufffd\ufffd", name: 'invalid continuation byte - 0xc2c0' },
 
         // Out of range
-        { bytes: [ 0xf4, 0x90, 0x80, 0x80 ], reason: 'out-of-range', name: 'out of range' },
+        { bytes: [ 0xf4, 0x90, 0x80, 0x80 ], reason: outOfRange, ignored: "", replaced: "\ufffd", name: 'out of range' },
     ];
 
     BadUTF.forEach(function(test) {
         it('toUtf8String - ' + test.name, function() {
+            // Check the string using the ignoreErrors conversion
+            const ignored = ethers.utils.toUtf8String(test.bytes, ethers.utils.Utf8ErrorFuncs.ignore);
+            assert.equal(ignored, test.ignored, "ignoring errors matches");
+
+            // Check the string using the replaceErrors conversion
+            const replaced = ethers.utils.toUtf8String(test.bytes, ethers.utils.Utf8ErrorFuncs.replace);
+            assert.equal(replaced, test.replaced, "replaced errors matches");
+
+            // Check the string throws the correct error during conversion
             assert.throws(function() {
                 let result = ethers.utils.toUtf8String(test.bytes);
                 console.log('Result', result);
             }, function(error: Error) {
-                return (error.message.split(';').pop().trim() === test.reason)
+                return (error.message.split(";").pop().split("(")[0].trim() === test.reason)
             }, test.name);
         });
     });