From 39129d0ec18aed689b015933e5026b215b7a5873 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 6 Mar 2019 02:36:09 +0200 Subject: [PATCH] Add base64 and tests for it. Adopt multibase to be able to use base64. Fix base32 typos. --- libp2p.nimble | 1 + libp2p/base32.nim | 29 ++--- libp2p/base64.nim | 238 ++++++++++++++++++++++++++++++++++++++++ libp2p/multibase.nim | 77 +++++++++++-- tests/testbase64.nim | 162 +++++++++++++++++++++++++++ tests/testmultibase.nim | 72 ++++++------ 6 files changed, 518 insertions(+), 61 deletions(-) create mode 100644 libp2p/base64.nim create mode 100644 tests/testbase64.nim diff --git a/libp2p.nimble b/libp2p.nimble index 257df01c1..f084e0da2 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -15,6 +15,7 @@ task test, "Runs the test suite": exec "nim c -r tests/testvarint" exec "nim c -r tests/testbase58" exec "nim c -r tests/testbase32" + exec "nim c -r tests/testbase64" exec "nim c -r tests/testmultiaddress" exec "nim c -r tests/testmultihash" exec "nim c -r tests/testmultibase" diff --git a/libp2p/base32.nim b/libp2p/base32.nim index 0a3fa7c33..e48f12dbe 100644 --- a/libp2p/base32.nim +++ b/libp2p/base32.nim @@ -22,23 +22,23 @@ type encode*: array[32, uint8] Base32Upper* = object - ## Type to use RFC4868 alphabet in uppercase without padding + ## Type to use RFC4648 alphabet in uppercase without padding Base32Lower* = object - ## Type to use RFC4868 alphabet in lowercase without padding + ## Type to use RFC4648 alphabet in lowercase without padding Base32UpperPad* = object - ## Type to use RFC4868 alphabet in uppercase with padding + ## Type to use RFC4648 alphabet in uppercase with padding Base32LowerPad* = object - ## Type to use RFC4868 alphabet in lowercase with padding + ## Type to use RFC4648 alphabet in lowercase with padding HexBase32Upper* = object - ## Type to use RFC4868-HEX alphabet in uppercase without padding + ## Type to use RFC4648-HEX alphabet in uppercase without padding HexBase32Lower* = object - ## Type to use RFC4868-HEX alphabet in lowercase without padding + ## Type to use RFC4648-HEX alphabet in lowercase without padding HexBase32UpperPad* = object - ## Type to use RFC4868-HEX alphabet in uppercase with padding + ## Type to use RFC4648-HEX alphabet in uppercase with padding HexBase32LowerPad* = object - ## Type to use RFC4868-HEX alphabet in lowercase with padding + ## Type to use RFC4648-HEX alphabet in lowercase with padding Base32* = Base32Upper - ## By default we are using RFC4868 alphabet in uppercase without padding + ## By default we are using RFC4648 alphabet in uppercase without padding Base32PadTypes* = Base32UpperPad | Base32LowerPad | HexBase32UpperPad | HexBase32LowerPad ## All types with padding support @@ -240,12 +240,10 @@ proc decode*[T: byte|char](btype: typedesc[Base32Types], instr: openarray[T], for j in 0..<8: if (cast[byte](instr[i + j]) and 0x80'u8) != 0: outlen = 0 - zeroMem(addr outbytes[0], i + 8) return Base32Status.Incorrect - let ch = alphabet.decode[int8(instr[i + j])] + let ch = alphabet.decode[cast[int8](instr[i + j])] if ch == -1: outlen = 0 - zeroMem(addr outbytes[0], i + 8) return Base32Status.Incorrect buffer[j] = cast[byte](ch) discard convert8to5(buffer, outbytes.toOpenArray(k, k + 4), 8) @@ -256,18 +254,15 @@ proc decode*[T: byte|char](btype: typedesc[Base32Types], instr: openarray[T], if reminder != 0: if reminder == 1 or reminder == 3 or reminder == 6: outlen = 0 - zeroMem(addr outbytes[0], i + 8) return Base32Status.Incorrect for j in 0.. len(outbytes): + outlen = length + return Base64Status.Overrun + + var inlen = len(instr) + when (btype is Base64PadTypes): + for i in countdown(inlen - 1, 0): + if instr[i] != '=': + break + dec(inlen) + + let reminder = inlen mod 4 + let limit = inlen - reminder + var buffer: array[4, byte] + var i, k: int + while i < limit: + for j in 0..<4: + if (cast[byte](instr[i + j]) and 0x80'u8) != 0: + outlen = 0 + zeroMem(addr outbytes[0], i + 3) + return Base64Status.Incorrect + let ch = alphabet.decode[cast[int8](instr[i + j])] + if ch == -1: + outlen = 0 + zeroMem(addr outbytes[0], i + 3) + return Base64Status.Incorrect + buffer[j] = cast[byte](ch) + outbytes[k] = cast[byte]((buffer[0] shl 2) or (buffer[1] shr 4)) + inc(k) + outbytes[k] = cast[byte]((buffer[1] shl 4) or (buffer[2] shr 2)) + inc(k) + outbytes[k] = cast[byte]((buffer[2] shl 6) or buffer[3]) + inc(k) + i += 4 + + if reminder > 0: + if reminder == 1: + outlen = 0 + return Base64Status.Incorrect + + for j in 0.. 1: + outbytes[k] = cast[byte]((buffer[0] shl 2) or (buffer[1] shr 4)) + inc(k) + if reminder > 2: + outbytes[k] = cast[byte]((buffer[1] shl 4) or (buffer[2] shr 2)) + inc(k) + + outlen = k + result = Base64Status.Success + +proc decode*[T: byte|char](btype: typedesc[Base64Types], + instr: openarray[T]): seq[byte] = + ## Decode BASE64 string ``instr`` and return sequence of bytes as result. + if len(instr) == 0: + result = newSeq[byte]() + else: + var length = 0 + result = newSeq[byte](btype.decodedLength(len(instr))) + if btype.decode(instr, result, length) == Base64Status.Success: + result.setLen(length) + else: + raise newException(Base64Error, "Incorrect base64 string") diff --git a/libp2p/multibase.nim b/libp2p/multibase.nim index fec965a51..954228e9c 100644 --- a/libp2p/multibase.nim +++ b/libp2p/multibase.nim @@ -10,10 +10,9 @@ ## This module implements MultiBase. ## ## TODO: -## 1. base64 -## 2. base32z +## 1. base32z import tables, strutils -import base32, base58 +import base32, base58, base64 type MultibaseStatus* {.pure.} = enum @@ -28,7 +27,7 @@ type outbytes: var openarray[char], outlen: var int): MultibaseStatus {.nimcall.} MBCodeSize = proc(length: int): int {.nimcall.} - + MBCodec = object code: char name: string @@ -108,6 +107,15 @@ proc b58ce(r: Base58Status): MultibaseStatus {.inline.} = elif r == Base58Status.Success: result = MultibaseStatus.Success +proc b64ce(r: Base64Status): MultibaseStatus {.inline.} = + result = MultiBaseStatus.Error + if r == Base64Status.Incorrect: + result = MultibaseStatus.Incorrect + elif r == Base64Status.Overrun: + result = MultiBaseStatus.Overrun + elif r == Base64Status.Success: + result = MultibaseStatus.Success + proc b32hd(inbytes: openarray[char], outbytes: var openarray[byte], outlen: var int): MultibaseStatus = @@ -216,6 +224,51 @@ proc b58be(inbytes: openarray[byte], proc b58el(length: int): int = Base58.encodedLength(length) proc b58dl(length: int): int = Base58.decodedLength(length) +proc b64el(length: int): int = Base64.encodedLength(length) +proc b64dl(length: int): int = Base64.decodedLength(length) +proc b64pel(length: int): int = Base64Pad.encodedLength(length) +proc b64pdl(length: int): int = Base64Pad.decodedLength(length) + +proc b64e(inbytes: openarray[byte], + outbytes: var openarray[char], + outlen: var int): MultibaseStatus = + result = b64ce(Base64.encode(inbytes, outbytes, outlen)) + +proc b64d(inbytes: openarray[char], + outbytes: var openarray[byte], + outlen: var int): MultibaseStatus = + result = b64ce(Base64.decode(inbytes, outbytes, outlen)) + +proc b64pe(inbytes: openarray[byte], + outbytes: var openarray[char], + outlen: var int): MultibaseStatus = + result = b64ce(Base64Pad.encode(inbytes, outbytes, outlen)) + +proc b64pd(inbytes: openarray[char], + outbytes: var openarray[byte], + outlen: var int): MultibaseStatus = + result = b64ce(Base64Pad.decode(inbytes, outbytes, outlen)) + +proc b64ue(inbytes: openarray[byte], + outbytes: var openarray[char], + outlen: var int): MultibaseStatus = + result = b64ce(Base64Url.encode(inbytes, outbytes, outlen)) + +proc b64ud(inbytes: openarray[char], + outbytes: var openarray[byte], + outlen: var int): MultibaseStatus = + result = b64ce(Base64Url.decode(inbytes, outbytes, outlen)) + +proc b64upe(inbytes: openarray[byte], + outbytes: var openarray[char], + outlen: var int): MultibaseStatus = + result = b64ce(Base64UrlPad.encode(inbytes, outbytes, outlen)) + +proc b64upd(inbytes: openarray[char], + outbytes: var openarray[byte], + outlen: var int): MultibaseStatus = + result = b64ce(Base64UrlPad.decode(inbytes, outbytes, outlen)) + const MultibaseCodecs = [ MBCodec(name: "identity", code: chr(0x00), @@ -262,10 +315,18 @@ const MBCodec(name: "base58btc", code: 'z', decr: b58bd, encr: b58be, decl: b58dl, encl: b58el ), - MBCodec(name: "base64", code: 'm'), - MBCodec(name: "base64pad", code: 'M'), - MBCodec(name: "base64url", code: 'u'), - MBCodec(name: "base64urlpad", code: 'U') + MBCodec(name: "base64", code: 'm', + decr: b64d, encr: b64e, decl: b64dl, encl: b64el + ), + MBCodec(name: "base64pad", code: 'M', + decr: b64pd, encr: b64pe, decl: b64pdl, encl: b64pel + ), + MBCodec(name: "base64url", code: 'u', + decr: b64ud, encr: b64ue, decl: b64dl, encl: b64el + ), + MBCodec(name: "base64urlpad", code: 'U', + decr: b64upd, encr: b64upe, decl: b64pdl, encl: b64pel + ) ] proc initMultiBaseCodeTable(): Table[char, MBCodec] {.compileTime.} = diff --git a/tests/testbase64.nim b/tests/testbase64.nim new file mode 100644 index 000000000..61d5b5d0b --- /dev/null +++ b/tests/testbase64.nim @@ -0,0 +1,162 @@ +import unittest +import ../libp2p/base64 + +const TVBasePadding = [ + ["f", "Zg=="], + ["fo", "Zm8="], + ["foo", "Zm9v"], + ["foob", "Zm9vYg=="], + ["fooba", "Zm9vYmE="], + ["foobar", "Zm9vYmFy"] +] + +const TVBaseNoPadding = [ + ["f", "Zg"], + ["fo", "Zm8"], + ["foo", "Zm9v"], + ["foob", "Zm9vYg"], + ["fooba", "Zm9vYmE"], + ["foobar", "Zm9vYmFy"] +] + +suite "BASE64 encoding test suite": + test "Empty seq/string test": + var empty1 = newSeq[byte]() + var empty2 = "" + var encoded = newString(16) + var decoded = newSeq[byte](16) + + var o1, o2, o3, o4: int + var e1 = Base64.encode(empty1) + var e2 = Base64Url.encode(empty1) + var e3 = Base64Pad.encode(empty1) + var e4 = Base64UrlPad.encode(empty1) + check: + Base64.encode(empty1, encoded, o1) == Base64Status.Success + Base64Url.encode(empty1, encoded, o2) == Base64Status.Success + Base64Pad.encode(empty1, encoded, o3) == Base64Status.Success + Base64UrlPad.encode(empty1, encoded, o4) == Base64Status.Success + len(e1) == 0 + len(e2) == 0 + len(e3) == 0 + len(e4) == 0 + o1 == 0 + o2 == 0 + o3 == 0 + o4 == 0 + var d1 = Base64.decode("") + var d2 = Base64Url.decode("") + var d3 = Base64Pad.decode("") + var d4 = Base64UrlPad.decode("") + check: + Base64.decode(empty2, decoded, o1) == Base64Status.Success + Base64Url.decode(empty2, decoded, o2) == Base64Status.Success + Base64Pad.decode(empty2, decoded, o3) == Base64Status.Success + Base64UrlPad.decode(empty2, decoded, o4) == Base64Status.Success + len(d1) == 0 + len(d2) == 0 + len(d3) == 0 + len(d4) == 0 + o1 == 0 + o2 == 0 + o3 == 0 + o4 == 0 + + test "Zero test": + var s = newString(256) + for i in 0..255: + s[i] = 'A' + var buffer: array[256, byte] + for i in 0..255: + var a = Base64.encode(buffer.toOpenArray(0, i)) + var b = Base64.decode(a) + check b == buffer[0..i] + + test "Leading zero test": + var buffer: array[256, byte] + for i in 0..255: + buffer[255] = byte(i) + var a = Base64.encode(buffer) + var b = Base64.decode(a) + check: + equalMem(addr buffer[0], addr b[0], 256) == true + + test "BASE64 padding test vectors": + for item in TVBasePadding: + let plain = cast[seq[byte]](item[0]) + let expect = item[1] + var elen = 0 + var dlen = 0 + + var e1 = Base64Pad.encode(plain) + var e2 = newString(Base64Pad.encodedLength(len(plain))) + check: + Base64Pad.encode(plain, e2, elen) == Base64Status.Success + e2.setLen(elen) + check: + e1 == expect + e2 == expect + + var d1 = Base64Pad.decode(expect) + var d2 = newSeq[byte](Base64Pad.decodedLength(len(expect))) + check: + Base64Pad.decode(expect, d2, dlen) == Base64Status.Success + d2.setLen(dlen) + check: + d1 == plain + d2 == plain + + test "BASE64 no padding test vectors": + for item in TVBaseNoPadding: + let plain = cast[seq[byte]](item[0]) + let expect = item[1] + var elen = 0 + var dlen = 0 + + var e1 = Base64.encode(plain) + var e2 = newString(Base64.encodedLength(len(plain))) + check: + Base64.encode(plain, e2, elen) == Base64Status.Success + e2.setLen(elen) + check: + e1 == expect + e2 == expect + + var d1 = Base64.decode(expect) + var d2 = newSeq[byte](Base64.decodedLength(len(expect))) + check: + Base64.decode(expect, d2, dlen) == Base64Status.Success + d2.setLen(dlen) + check: + d1 == plain + d2 == plain + + test "Buffer Overrun test": + var encres = "" + var encsize = 0 + var decres: seq[byte] = @[] + var decsize = 0 + check: + Base64.encode([0'u8], encres, encsize) == Base64Status.Overrun + encsize == Base64.encodedLength(1) + Base64.decode("AA", decres, decsize) == Base64Status.Overrun + decsize == Base64.decodedLength(2) + + test "Incorrect test": + var decres = newSeq[byte](10) + var decsize = 0 + check: + Base64.decode("A", decres, decsize) == Base64Status.Incorrect + decsize == 0 + Base64.decode("AAAAA", decres, decsize) == Base64Status.Incorrect + decsize == 0 + Base64.decode("!", decres, decsize) == Base64Status.Incorrect + decsize == 0 + Base64.decode("!!", decres, decsize) == Base64Status.Incorrect + decsize == 0 + Base64.decode("AA==", decres, decsize) == Base64Status.Incorrect + decsize == 0 + Base64.decode("_-", decres, decsize) == Base64Status.Incorrect + decsize == 0 + Base64Url.decode("/+", decres, decsize) == Base64Status.Incorrect + decsize == 0 diff --git a/tests/testmultibase.nim b/tests/testmultibase.nim index 8a67ed86d..1679e8ccd 100644 --- a/tests/testmultibase.nim +++ b/tests/testmultibase.nim @@ -62,26 +62,26 @@ const GoTestVectors = [ "z36UQrhJq9fNDS7DiAHM9YXqDHMPfr4EMArvt", "Decentralize everything!!!" ], - # [ - # "base64", - # "mRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE", - # "Decentralize everything!!!" - # ], - # [ - # "base64url", - # "uRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE", - # "Decentralize everything!!!" - # ], - # [ - # "base64pad", - # "MRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE=", - # "Decentralize everything!!!" - # ], - # [ - # "base64urlpad", - # "URGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE=", - # "Decentralize everything!!!" - # ], + [ + "base64", + "mRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE", + "Decentralize everything!!!" + ], + [ + "base64url", + "uRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE", + "Decentralize everything!!!" + ], + [ + "base64pad", + "MRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE=", + "Decentralize everything!!!" + ], + [ + "base64urlpad", + "URGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE=", + "Decentralize everything!!!" + ], ] suite "MultiBase test suite": @@ -112,10 +112,10 @@ suite "MultiBase test suite": MultiBase.encode("base32padupper", plain) == "C" MultiBase.encode("base58btc", plain) == "z" MultiBase.encode("base58flickr", plain) == "Z" - # MultiBase.encode("base64", plain) == "m" - # MultiBase.encode("base64pad", plain) == "M" - # MultiBase.encode("base64url", plain) == "u" - # MultiBase.encode("base64urlpad", plain) == "U" + MultiBase.encode("base64", plain) == "m" + MultiBase.encode("base64pad", plain) == "M" + MultiBase.encode("base64url", plain) == "u" + MultiBase.encode("base64urlpad", plain) == "U" check: len(MultiBase.decode("\x00")) == 0 # len(MultiBase.decode("1")) == 0 @@ -134,10 +134,10 @@ suite "MultiBase test suite": len(MultiBase.decode("C")) == 0 len(MultiBase.decode("z")) == 0 len(MultiBase.decode("Z")) == 0 - # len(MultiBase.decode("m")) == 0 - # len(MultiBase.decode("M")) == 0 - # len(MultiBase.decode("u")) == 0 - # len(MultiBase.decode("U")) == 0 + len(MultiBase.decode("m")) == 0 + len(MultiBase.decode("M")) == 0 + len(MultiBase.decode("u")) == 0 + len(MultiBase.decode("U")) == 0 check: MultiBase.encode("identity", plain, enc, olens[0]) == MultiBaseStatus.Success @@ -243,14 +243,14 @@ suite "MultiBase test suite": olens[15] == 0 MultiBase.decode("Z", dec, olens[16]) == MultiBaseStatus.Success olens[16] == 0 - # MultiBase.decode("m", dec, olens[16]) == MultiBaseStatus.Success - # olens[16] == 0 - # MultiBase.decode("M", dec, olens[16]) == MultiBaseStatus.Success - # olens[16] == 0 - # MultiBase.decode("u", dec, olens[16]) == MultiBaseStatus.Success - # olens[16] == 0 - # MultiBase.decode("U", dec, olens[16]) == MultiBaseStatus.Success - # olens[16] == 0 + MultiBase.decode("m", dec, olens[16]) == MultiBaseStatus.Success + olens[16] == 0 + MultiBase.decode("M", dec, olens[16]) == MultiBaseStatus.Success + olens[16] == 0 + MultiBase.decode("u", dec, olens[16]) == MultiBaseStatus.Success + olens[16] == 0 + MultiBase.decode("U", dec, olens[16]) == MultiBaseStatus.Success + olens[16] == 0 test "go-multibase test vectors": for item in GoTestVectors: let encoding = item[0]