Noise (#90)

* Start ChaCha20Poly1305 integration (BearSSL)

* Add Curve25519 (BearSSL) required operations for noise

* Fix curve mulgen iterate/derive

* Fix misleading header

* Add chachapoly proper test

* Curve25519 integration tests (failing, something is wrong)

* Add few converters, finish c25519 integration tests

* Remove implicit converters

* removed internal globals

* Start noise implementation

* Fix public() using proper bear mulgen

* Noise protocol WIP

* Noise progress

* Add a quick nim version of HKDF

* Converted hkdf to iterator, useful for noise

* Noise protocol implementation progress

* Noise progress

* XX handshake almost there

* noise progress

* Noise, testing handshake with test vectors

* Noise handshake progress, still wrong somewhere!

* Noise handshake success!

* Full verified noise XX handshake completed

* Fix and rewrite test to be similar to switch one

* Start with connection upgrade

* Switch chachapoly to CT implementations

* Improve HKDF implementation

* Use a type insted of tuple for HandshakeResult

* Remove unnecessary Let

* More cosmetic fixes

* Properly check randomBytes result

* Fix chachapoly signature

* Noise full circle (altho dispatcher is nil cursed)

* Allow nil aads in chachapoly routines

* Noise implementation up to running full test

* Use bearssl HKDF as well

* Directly use bearssl rng for curve25519 keys

* Add a (disabled/no CI) noise interop test server

* WIP on fixing interop issues

* More fixes in noise implementation for interop

* bump chronos requirement (nimble)

* Add a chachapoly test for very small size payloads

* Noise, more tracing

* Add 2 properly working noise tests

* Fix payload packing, following the spec properly (and not go version but
rather rust)

* Sanity, replace discard with asyncCheck

* Small fixes and optimization

* Use stew endian2 rather then system endian module

* Update nimble deps (chronos)

* Minor cosmetic/code sanity fixes

* Noise, handle Nonce max

* Noise tests, make sure to close secured conns

* More polish, improve code readability too

* More polish and testing again which test fails

* Further polishing

* Restore noise tests

* Remove useless Future[void]

* Remove useless CipherState initializer

* add a proper read wait future in second noise test

* Remove noise generic secure implementation for now

* Few fixes to run eth2 sim

* Add more debug info in noise traces

* Merge size + payload write in sendEncryptedMessage

* Revert secure interface, add outgoing property directly in newNoise

* remove sendEncrypted and receiveEncrypted

* Use openarray in chachapoly and curve25519 helpers

libp2p.nimble

@@ -22,4 +22,4 @@ proc runTest(filename: string) =
 task test, "Runs the test suite":
   runTest "testnative"
   runTest "testdaemon"
-  runTest "testinterop"
+  runTest "testinterop"

libp2p/crypto/chacha20poly1305.nim

@@ -28,68 +28,73 @@ type
   ChaChaPolyNonce* = array[ChaChaPolyNonceSize, byte]
   ChaChaPolyTag* = array[ChaChaPolyTagSize, byte]
 
-proc intoChaChaPolyKey*(s: seq[byte]): ChaChaPolyKey =
+proc intoChaChaPolyKey*(s: openarray[byte]): ChaChaPolyKey =
   assert s.len == ChaChaPolyKeySize
   copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyKeySize)
 
-proc intoChaChaPolyNonce*(s: seq[byte]): ChaChaPolyNonce =
+proc intoChaChaPolyNonce*(s: openarray[byte]): ChaChaPolyNonce =
   assert s.len == ChaChaPolyNonceSize
   copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyNonceSize)
 
-proc intoChaChaPolyTag*(s: seq[byte]): ChaChaPolyTag =
+proc intoChaChaPolyTag*(s: openarray[byte]): ChaChaPolyTag =
   assert s.len == ChaChaPolyTagSize
   copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyTagSize)
-  
+   
 # bearssl allows us to use optimized versions
 # this is reconciled at runtime
 # we do this in the global scope / module init
 
 template fetchImpl: untyped =
   # try for the best first
-  var
+  let
-    chachapoly_native_impl {.inject.}: Poly1305Run = poly1305CtmulqGet()
+    chachapoly_native_impl {.inject.}: Poly1305Run = poly1305CtmulRun
-    chacha_native_impl {.inject.}: Chacha20Run = chacha20Sse2Get()
+    chacha_native_impl {.inject.}: Chacha20Run = chacha20CtRun
-
-  # fall back if not available
-  if chachapoly_native_impl == nil:
-    chachapoly_native_impl = poly1305CtmulRun
-
-  if chacha_native_impl == nil:
-    chacha_native_impl = chacha20CtRun
 
 proc encrypt*(_: type[ChaChaPoly],
-                 key: var ChaChaPolyKey,
+                 key: ChaChaPolyKey,
-                 nonce: var ChaChaPolyNonce,
+                 nonce: ChaChaPolyNonce,
                  tag: var ChaChaPolyTag,
                  data: var openarray[byte],
-                 aad: var openarray[byte]) =
+                 aad: openarray[byte]) =
   fetchImpl()
-  
+
+  let
+    ad = if aad.len > 0:
+           unsafeaddr aad[0]
+         else:
+           nil
+
   chachapoly_native_impl(
-    addr key[0],
+    unsafeaddr key[0],
-    addr nonce[0],
+    unsafeaddr nonce[0],
     addr data[0],
     data.len,
-    addr aad[0],
+    ad,
     aad.len,
     addr tag[0],
     chacha_native_impl,
     #[encrypt]# 1.cint)
 
 proc decrypt*(_: type[ChaChaPoly],
-                 key: var ChaChaPolyKey,
+                 key: ChaChaPolyKey,
-                 nonce: var ChaChaPolyNonce,
+                 nonce: ChaChaPolyNonce,
                  tag: var ChaChaPolyTag,
                  data: var openarray[byte],
-                 aad: var openarray[byte]) =
+                 aad: openarray[byte]) =
   fetchImpl()
+
+  let
+    ad = if aad.len > 0:
+          unsafeaddr aad[0]
+         else:
+           nil
   
   chachapoly_native_impl(
-    addr key[0],
+    unsafeaddr key[0],
-    addr nonce[0],
+    unsafeaddr nonce[0],
     addr data[0],
     data.len,
-    addr aad[0],
+    ad,
     aad.len,
     addr tag[0],
     chacha_native_impl,

libp2p/crypto/curve25519.nim

@@ -16,6 +16,7 @@
 # RFC @ https://tools.ietf.org/html/rfc7748
 
 import bearssl
+import nimcrypto/sysrand
 
 const
   Curve25519KeySize* = 32
@@ -24,10 +25,13 @@ type
   Curve25519* = object
   Curve25519Key* = array[Curve25519KeySize, byte]
   pcuchar = ptr cuchar
+  Curver25519RngError* = object of CatchableError
 
-proc intoCurve25519Key*(s: seq[byte]): Curve25519Key =
+proc intoCurve25519Key*(s: openarray[byte]): Curve25519Key =
   assert s.len == Curve25519KeySize
   copyMem(addr result[0], unsafeaddr s[0], Curve25519KeySize)
+
+proc getBytes*(key: Curve25519Key): seq[byte] = @key
   
 const
   ForbiddenCurveValues: array[12, Curve25519Key] = [
@@ -45,7 +49,7 @@ const
                 [219.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25],
         ]
 
-proc byteswap*(buf: var Curve25519Key) {.inline.} =
+proc byteswap(buf: var Curve25519Key) {.inline.} =
   for i in 0..<16:
     let
       x = buf[i]
@@ -97,3 +101,12 @@ proc mulgen*(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key)
 proc public*(private: Curve25519Key): Curve25519Key =
   Curve25519.mulgen(result, private)
 
+proc random*(_: type[Curve25519Key]): Curve25519Key =
+  var rng: BrHmacDrbgContext
+  let seeder = brPrngSeederSystem(nil)
+  brHmacDrbgInit(addr rng, addr sha256Vtable, nil, 0)
+  if seeder(addr rng.vtable) == 0:
+    raise newException(ValueError, "Could not seed RNG")
+  let defaultBrEc = brEcGetDefault()
+  if brEcKeygen(addr rng.vtable, defaultBrEc, nil, addr result[0], EC_curve25519) != Curve25519KeySize:
+    raise newException(Curver25519RngError, "Could not generate random data")

libp2p/crypto/hkdf.nim

@@ -0,0 +1,32 @@
+## Nim-LibP2P
+## Copyright (c) 2020 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+# https://tools.ietf.org/html/rfc5869
+
+import math
+import nimcrypto
+import bearssl
+
+type
+  BearHKDFContext {.importc: "br_hkdf_context", header: "bearssl_kdf.h".} = object
+  HKDFResult*[len: static int] = array[len, byte]
+
+proc br_hkdf_init(ctx: ptr BearHKDFContext; hashClass: ptr HashClass; salt: pointer; len: csize) {.importc: "br_hkdf_init", header: "bearssl_kdf.h".}
+proc br_hkdf_inject(ctx: ptr BearHKDFContext; ikm: pointer; len: csize) {.importc: "br_hkdf_inject", header: "bearssl_kdf.h".}
+proc br_hkdf_flip(ctx: ptr BearHKDFContext) {.importc: "br_hkdf_flip", header: "bearssl_kdf.h".}
+proc br_hkdf_produce(ctx: ptr BearHKDFContext; info: pointer; infoLen: csize; output: pointer; outputLen: csize) {.importc: "br_hkdf_produce", header: "bearssl_kdf.h".}
+
+proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openarray[byte]; outputs: var openarray[HKDFResult[len]]) =
+  var
+    ctx: BearHKDFContext
+  br_hkdf_init(addr ctx, addr sha256Vtable, if salt.len > 0: unsafeaddr salt[0] else: nil, salt.len)
+  br_hkdf_inject(addr ctx, if ikm.len > 0: unsafeaddr ikm[0] else: nil, ikm.len)
+  br_hkdf_flip(addr ctx)
+  for i in 0..outputs.high:
+    br_hkdf_produce(addr ctx, if info.len > 0: unsafeaddr info[0] else: nil, info.len, addr outputs[i][0], outputs[i].len)

libp2p/protocols/secure/noise.nim

@@ -0,0 +1,519 @@
+## Nim-LibP2P
+## Copyright (c) 2020 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import chronos
+import chronicles
+import random
+import stew/[endians2, byteutils]
+import nimcrypto/[utils, sysrand, sha2, hmac]
+import ../../connection
+import ../../peer
+import ../../peerinfo
+import ../../protobuf/minprotobuf
+import secure,
+       ../../crypto/[crypto, chacha20poly1305, curve25519, hkdf],
+       ../../stream/bufferstream
+
+logScope:
+  topic = "Noise"
+
+const
+  # https://godoc.org/github.com/libp2p/go-libp2p-noise#pkg-constants
+  NoiseCodec* = "/noise"
+  
+  PayloadString = "noise-libp2p-static-key:"
+
+  ProtocolXXName = "Noise_XX_25519_ChaChaPoly_SHA256"
+
+  # Empty is a special value which indicates k has not yet been initialized.
+  EmptyKey: ChaChaPolyKey = [0.byte, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+  NonceMax = uint64.high - 1 # max is reserved
+
+type
+  KeyPair = object
+    privateKey: Curve25519Key
+    publicKey: Curve25519Key
+  
+  # https://noiseprotocol.org/noise.html#the-cipherstate-object
+  CipherState = object
+    k: ChaChaPolyKey
+    n: uint64
+
+  # https://noiseprotocol.org/noise.html#the-symmetricstate-object
+  SymmetricState = object
+    cs: CipherState
+    ck: ChaChaPolyKey
+    h: MDigest[256]
+
+  # https://noiseprotocol.org/noise.html#the-handshakestate-object
+  HandshakeState = object
+    ss: SymmetricState
+    s: KeyPair
+    e: KeyPair
+    rs: Curve25519Key
+    re: Curve25519Key
+
+  HandshakeResult = object
+    cs1: CipherState
+    cs2: CipherState
+    remoteP2psecret: seq[byte]
+    rs: Curve25519Key
+  
+  Noise* = ref object of Secure
+    localPrivateKey: PrivateKey
+    localPublicKey: PublicKey
+    noisePrivateKey: Curve25519Key
+    noisePublicKey: Curve25519Key
+    commonPrologue: seq[byte]
+    outgoing: bool
+
+  NoiseConnection* = ref object of SecureConn
+    readCs: CipherState
+    writeCs: CipherState
+
+  NoiseHandshakeError* = object of CatchableError
+  NoiseDecryptTagError* = object of CatchableError
+  NoiseOversizedPayloadError* = object of CatchableError
+  NoiseNonceMaxError* = object of CatchableError # drop connection on purpose
+
+# Utility
+
+proc genKeyPair(): KeyPair =
+  result.privateKey = Curve25519Key.random()
+  result.publicKey = result.privateKey.public()
+    
+proc hashProtocol(name: string): MDigest[256] =
+  # If protocol_name is less than or equal to HASHLEN bytes in length,
+  # sets h equal to protocol_name with zero bytes appended to make HASHLEN bytes.
+  # Otherwise sets h = HASH(protocol_name).
+
+  if name.len <= 32:
+    result.data[0..name.high] = name.toBytes
+  else:
+    result = sha256.digest(name)
+
+proc dh(priv: Curve25519Key, pub: Curve25519Key): Curve25519Key =
+  Curve25519.mul(result, pub, priv)
+
+# Cipherstate
+
+proc hasKey(cs: CipherState): bool =
+  cs.k != EmptyKey
+
+proc encryptWithAd(state: var CipherState, ad, data: openarray[byte]): seq[byte] =
+  var
+    tag: ChaChaPolyTag
+    nonce: ChaChaPolyNonce
+    np = cast[ptr uint64](addr nonce[4])
+  np[] = state.n
+  result = @data
+  ChaChaPoly.encrypt(state.k, nonce, tag, result, ad)
+  inc state.n
+  if state.n > NonceMax:
+    raise newException(NoiseNonceMaxError, "Noise max nonce value reached")
+  result &= tag
+  trace "encryptWithAd", tag = byteutils.toHex(tag), data = byteutils.toHex(result), nonce = state.n - 1
+
+proc decryptWithAd(state: var CipherState, ad, data: openarray[byte]): seq[byte] =
+  var
+    tagIn = data[^ChaChaPolyTag.len..data.high].intoChaChaPolyTag
+    tagOut = tagIn
+    nonce: ChaChaPolyNonce
+    np = cast[ptr uint64](addr nonce[4])
+  np[] = state.n
+  result = data[0..(data.high - ChaChaPolyTag.len)]
+  ChaChaPoly.decrypt(state.k, nonce, tagOut, result, ad)
+  trace "decryptWithAd", tagIn = byteutils.toHex(tagIn), tagOut=byteutils.toHex(tagOut), nonce = state.n
+  if tagIn != tagOut:
+    error "decryptWithAd failed", data = byteutils.toHex(data)
+    raise newException(NoiseDecryptTagError, "decryptWithAd failed tag authentication.")
+  inc state.n
+  if state.n > NonceMax:
+    raise newException(NoiseNonceMaxError, "Noise max nonce value reached")
+
+# Symmetricstate
+
+proc init(_: type[SymmetricState]): SymmetricState =
+  result.h = ProtocolXXName.hashProtocol
+  result.ck = result.h.data.intoChaChaPolyKey
+  result.cs = CipherState(k: EmptyKey)
+
+proc mixKey(ss: var SymmetricState, ikm: ChaChaPolyKey) =
+  var
+    temp_keys: array[2, ChaChaPolyKey]
+  sha256.hkdf(ss.ck, ikm, [], temp_keys)
+  ss.ck = temp_keys[0]
+  ss.cs = CipherState(k: temp_keys[1])
+  trace "mixKey", key = ss.cs.k
+
+proc mixHash(ss: var SymmetricState; data: openarray[byte]) =
+  var ctx: sha256
+  ctx.init()
+  ctx.update(ss.h.data)
+  ctx.update(data)
+  ss.h = ctx.finish()
+  trace "mixHash", hash = ss.h.data
+
+# We might use this for other handshake patterns/tokens
+proc mixKeyAndHash(ss: var SymmetricState; ikm: openarray[byte]) {.used.} =
+  var
+    temp_keys: array[3, ChaChaPolyKey]
+  sha256.hkdf(ss.ck, ikm, [], temp_keys)
+  ss.ck = temp_keys[0]
+  ss.mixHash(temp_keys[1])
+  ss.cs = CipherState(k: temp_keys[2])
+
+proc encryptAndHash(ss: var SymmetricState, data: openarray[byte]): seq[byte] =
+  # according to spec if key is empty leave plaintext
+  if ss.cs.hasKey:
+    result = ss.cs.encryptWithAd(ss.h.data, data)
+  else:
+    result = @data
+  ss.mixHash(result)
+
+proc decryptAndHash(ss: var SymmetricState, data: openarray[byte]): seq[byte] =
+  # according to spec if key is empty leave plaintext
+  if ss.cs.hasKey:
+    result = ss.cs.decryptWithAd(ss.h.data, data)
+  else:
+    result = @data
+  ss.mixHash(data)
+
+proc split(ss: var SymmetricState): tuple[cs1, cs2: CipherState] =
+  var
+    temp_keys: array[2, ChaChaPolyKey]
+  sha256.hkdf(ss.ck, [], [], temp_keys)
+  return (CipherState(k: temp_keys[0]), CipherState(k: temp_keys[1]))
+
+proc init(_: type[HandshakeState]): HandshakeState =
+  result.ss = SymmetricState.init()
+  
+template write_e: untyped =
+  trace "noise write e"
+  # Sets e (which must be empty) to GENERATE_KEYPAIR(). Appends e.public_key to the buffer. Calls MixHash(e.public_key).
+  hs.e = genKeyPair()
+  msg &= hs.e.publicKey
+  hs.ss.mixHash(hs.e.publicKey)
+
+template write_s: untyped =
+  trace "noise write s"
+  # Appends EncryptAndHash(s.public_key) to the buffer.
+  msg &= hs.ss.encryptAndHash(hs.s.publicKey)
+
+template dh_ee: untyped =
+  trace "noise dh ee"
+  # Calls MixKey(DH(e, re)).
+  hs.ss.mixKey(dh(hs.e.privateKey, hs.re))
+
+template dh_es: untyped =
+  trace "noise dh es"
+  # Calls MixKey(DH(e, rs)) if initiator, MixKey(DH(s, re)) if responder.
+  when initiator:
+    hs.ss.mixKey(dh(hs.e.privateKey, hs.rs))
+  else:
+    hs.ss.mixKey(dh(hs.s.privateKey, hs.re))
+
+template dh_se: untyped =
+  trace "noise dh se"
+  # Calls MixKey(DH(s, re)) if initiator, MixKey(DH(e, rs)) if responder.
+  when initiator:
+    hs.ss.mixKey(dh(hs.s.privateKey, hs.re))
+  else:
+    hs.ss.mixKey(dh(hs.e.privateKey, hs.rs))
+
+# might be used for other token/handshakes
+template dh_ss: untyped {.used.} =
+  trace "noise dh ss"
+  # Calls MixKey(DH(s, rs)).
+  hs.ss.mixKey(dh(hs.s.privateKey, hs.rs))
+
+template read_e: untyped =
+  trace "noise read e", size = msg.len
+
+  if msg.len < Curve25519Key.len:
+    raise newException(NoiseHandshakeError, "Noise E, expected more data")
+
+  # Sets re (which must be empty) to the next DHLEN bytes from the message. Calls MixHash(re.public_key).
+  hs.re[0..Curve25519Key.high] = msg[0..Curve25519Key.high]
+  msg = msg[Curve25519Key.len..msg.high]
+  hs.ss.mixHash(hs.re)
+
+template read_s: untyped =
+  trace "noise read s", size = msg.len
+  # Sets temp to the next DHLEN + 16 bytes of the message if HasKey() == True, or to the next DHLEN bytes otherwise.
+  # Sets rs (which must be empty) to DecryptAndHash(temp).
+  let
+    temp =
+      if hs.ss.cs.hasKey:
+        if msg.len < Curve25519Key.len + ChaChaPolyTag.len:
+          raise newException(NoiseHandshakeError, "Noise S, expected more data")
+        msg[0..Curve25519Key.high + ChaChaPolyTag.len]
+      else:
+        if msg.len < Curve25519Key.len:
+          raise newException(NoiseHandshakeError, "Noise S, expected more data")
+        msg[0..Curve25519Key.high]
+  msg = msg[temp.len..msg.high]
+  let plain = hs.ss.decryptAndHash(temp)
+  hs.rs[0..Curve25519Key.high] = plain
+
+proc receiveHSMessage(sconn: Connection): Future[seq[byte]] {.async.} =
+  var besize: array[2, byte]
+  await sconn.readExactly(addr besize[0], 2)
+  let size = uint16.fromBytesBE(besize).int
+  trace "receiveHSMessage", size
+  return await sconn.read(size)
+
+proc sendHSMessage(sconn: Connection; buf: seq[byte]) {.async.} =
+  var
+    lesize = buf.len.uint16
+    besize = lesize.toBytesBE
+  trace "sendHSMessage", size = lesize
+  await sconn.write(besize[0].addr, besize.len)
+  await sconn.write(buf)
+
+proc packNoisePayload(payload: openarray[byte]): seq[byte] =
+  let
+    noiselen = rand(2..31)
+    plen = payload.len.uint16
+
+  var
+    noise = newSeq[byte](noiselen)
+  if randomBytes(noise) != noiselen:
+    raise newException(NoiseHandshakeError, "Failed to generate randomBytes")
+
+  result &= plen.toBytesBE
+  result &= payload
+  result &= noise
+
+  if result.len > uint16.high.int:
+    raise newException(NoiseOversizedPayloadError, "Trying to send an unsupported oversized payload over Noise")
+ 
+  trace "packed noise payload", inSize = payload.len, outSize = result.len
+
+proc unpackNoisePayload(payload: var seq[byte]) =
+  let
+    besize = payload[0..1]
+    size = uint16.fromBytesBE(besize).int
+
+  if size > (payload.len - 2):
+    raise newException(NoiseOversizedPayloadError, "Received a wrong payload size")
+ 
+  payload = payload[2..^((payload.len - size) - 1)]
+
+  trace "unpacked noise payload", size = payload.len
+
+proc handshakeXXOutbound(p: Noise, conn: Connection, p2pProof: ProtoBuffer): Future[HandshakeResult] {.async.} =
+  const initiator = true
+
+  var
+    hs = HandshakeState.init()
+    p2psecret = p2pProof.buffer
+
+  hs.ss.mixHash(p.commonPrologue)
+  hs.s.privateKey = p.noisePrivateKey
+  hs.s.publicKey = p.noisePublicKey
+
+  # -> e
+  var msg: seq[byte]
+
+  write_e()
+
+  # IK might use this btw!
+  msg &= hs.ss.encryptAndHash(@[])
+
+  await conn.sendHSMessage(msg)
+
+  # <- e, ee, s, es
+
+  msg = await conn.receiveHSMessage()
+
+  read_e()
+  dh_ee()
+  read_s()
+  dh_es()
+
+  var remoteP2psecret = hs.ss.decryptAndHash(msg)
+  unpackNoisePayload(remoteP2psecret)
+
+  # -> s, se
+
+  msg.setLen(0)
+
+  write_s()
+  dh_se()
+
+  # last payload must follow the ecrypted way of sending
+  var packed = packNoisePayload(p2psecret)
+  msg &= hs.ss.encryptAndHash(packed)
+
+  await conn.sendHSMessage(msg)
+  
+  let (cs1, cs2) = hs.ss.split()
+  return HandshakeResult(cs1: cs1, cs2: cs2, remoteP2psecret: remoteP2psecret, rs: hs.rs)
+
+proc handshakeXXInbound(p: Noise, conn: Connection, p2pProof: ProtoBuffer): Future[HandshakeResult] {.async.} =
+  const initiator = false
+
+  var
+    hs = HandshakeState.init()
+    p2psecret = p2pProof.buffer
+
+  hs.ss.mixHash(p.commonPrologue)
+  hs.s.privateKey = p.noisePrivateKey
+  hs.s.publicKey = p.noisePublicKey
+
+  # -> e
+
+  var msg = await conn.receiveHSMessage()
+
+  read_e()
+
+  # we might use this early data one day, keeping it here for clarity
+  let earlyData {.used.} = hs.ss.decryptAndHash(msg)
+
+  # <- e, ee, s, es
+
+  msg.setLen(0)
+
+  write_e()
+  dh_ee()
+  write_s()
+  dh_es()
+
+  var packedSecret = packNoisePayload(p2psecret)
+  msg &= hs.ss.encryptAndHash(packedSecret)
+
+  await conn.sendHSMessage(msg)
+
+  # -> s, se
+
+  msg = await conn.receiveHSMessage()
+
+  read_s()
+  dh_se()
+
+  var remoteP2psecret = hs.ss.decryptAndHash(msg)
+  unpackNoisePayload(remoteP2psecret)
+
+  let (cs1, cs2) = hs.ss.split()
+  return HandshakeResult(cs1: cs1, cs2: cs2, remoteP2psecret: remoteP2psecret, rs: hs.rs)
+
+method readMessage(sconn: NoiseConnection): Future[seq[byte]] {.async.} =
+  try:
+    var besize: array[2, byte]
+    await sconn.readExactly(addr besize[0], 2)
+    let size = uint16.fromBytesBE(besize).int
+    trace "receiveEncryptedMessage", size, peer = $sconn.peerInfo
+    if size == 0:
+      return @[]
+    let
+      cipher = await sconn.read(size)
+    var plain = sconn.readCs.decryptWithAd([], cipher)
+    unpackNoisePayload(plain)
+    return plain
+  except AsyncStreamIncompleteError:
+    trace "Connection dropped while reading"
+  except AsyncStreamReadError:
+    trace "Error reading from connection"
+
+method writeMessage(sconn: NoiseConnection, message: seq[byte]): Future[void] {.async.} =
+  try:
+    let
+      packed = packNoisePayload(message)
+      cipher = sconn.writeCs.encryptWithAd([], packed)
+    var
+      lesize = cipher.len.uint16
+      besize = lesize.toBytesBE
+      outbuf = newSeqOfCap[byte](cipher.len + 2)
+    trace "sendEncryptedMessage", size = lesize, peer = $sconn.peerInfo
+    outbuf &= besize
+    outbuf &= cipher
+    await sconn.write(outbuf)
+  except AsyncStreamWriteError:
+    trace "Could not write to connection"
+
+method handshake*(p: Noise, conn: Connection, initiator: bool = false): Future[SecureConn] {.async.} =
+  trace "Starting Noise handshake", initiator
+
+  # https://github.com/libp2p/specs/tree/master/noise#libp2p-data-in-handshake-messages
+  let
+    signedPayload = p.localPrivateKey.sign(PayloadString.toBytes & p.noisePublicKey.getBytes)
+    
+  var
+    libp2pProof = initProtoBuffer()
+
+  libp2pProof.write(initProtoField(1, p.localPublicKey))
+  libp2pProof.write(initProtoField(2, signedPayload))
+  # data field also there but not used!
+  libp2pProof.finish()
+
+  let handshakeRes =
+    if initiator:
+      await handshakeXXOutbound(p, conn, libp2pProof)
+    else:
+      await handshakeXXInbound(p, conn, libp2pProof)
+
+  var
+    remoteProof = initProtoBuffer(handshakeRes.remoteP2psecret)
+    remotePubKey: PublicKey
+    remoteSig: Signature
+  if remoteProof.getValue(1, remotePubKey) <= 0:
+    raise newException(NoiseHandshakeError, "Failed to deserialize remote public key.")
+  if remoteProof.getValue(2, remoteSig) <= 0:
+    raise newException(NoiseHandshakeError, "Failed to deserialize remote public key.")
+
+  let verifyPayload = PayloadString.toBytes & handshakeRes.rs.getBytes
+  if not remoteSig.verify(verifyPayload, remotePubKey):
+    raise newException(NoiseHandshakeError, "Noise handshake signature verify failed.")
+  else:
+    trace "Remote signature verified"
+ 
+  if initiator and not isNil(conn.peerInfo):
+    let pid = PeerID.init(remotePubKey)
+    if not conn.peerInfo.peerId.validate():
+      raise newException(NoiseHandshakeError, "Failed to validate peerId.")
+    if pid != conn.peerInfo.peerId:
+      raise newException(NoiseHandshakeError, "Noise handshake, peer infos don't match! " & $pid & " != " & $conn.peerInfo.peerId)
+
+  var secure = new NoiseConnection
+  secure.stream = conn
+  secure.closeEvent = newAsyncEvent()
+  secure.peerInfo = PeerInfo.init(remotePubKey)
+  if initiator:
+    secure.readCs = handshakeRes.cs2
+    secure.writeCs = handshakeRes.cs1
+  else:
+    secure.readCs = handshakeRes.cs1
+    secure.writeCs = handshakeRes.cs2
+
+  debug "Noise handshake completed!"
+
+  return secure
+ 
+method init*(p: Noise) {.gcsafe.} =
+  procCall Secure(p).init()
+  p.codec = NoiseCodec
+
+method secure*(p: Noise, conn: Connection): Future[Connection] {.async, gcsafe.} =
+  try:
+    result = await p.handleConn(conn, p.outgoing)
+  except CatchableError as exc:
+    warn "securing connection failed", msg = exc.msg
+    if not conn.closed():
+      await conn.close()
+  
+proc newNoise*(privateKey: PrivateKey; outgoing: bool = true; commonPrologue: seq[byte] = @[]): Noise =
+  new result
+  result.outgoing = outgoing
+  result.localPrivateKey = privateKey
+  result.localPublicKey = privateKey.getKey()
+  discard randomBytes(result.noisePrivateKey)
+  result.noisePublicKey = result.noisePrivateKey.public()
+  result.commonPrologue = commonPrologue
+  result.init()

libp2p/protocols/secure/secure.nim

@@ -68,7 +68,7 @@ method init*(s: Secure) {.gcsafe.} =
   proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
     trace "handling connection"
     try:
-      asyncCheck s.handleConn(conn)
+      asyncCheck s.handleConn(conn, false)
       trace "connection secured"
     except CatchableError as exc:
       if not conn.closed():

libp2p/switch.nim

@@ -181,7 +181,7 @@ proc upgradeOutgoing(s: Switch, conn: Connection): Future[Connection] {.async, g
   s.connections[conn.peerInfo.id] = result
 
 proc upgradeIncoming(s: Switch, conn: Connection) {.async, gcsafe.} =
-  trace "upgrading incoming connection"
+  trace "upgrading incoming connection", conn = $conn
   let ms = newMultistream()
 
   # secure incoming connections

libp2p/transports/tcptransport.nim

@@ -40,7 +40,7 @@ proc connCb(server: StreamServer,
             client: StreamTransport) {.async, gcsafe.} =
   trace "incomming connection for", address = $client.remoteAddress
   let t: Transport = cast[Transport](server.udata)
-  discard t.connHandler(server, client)
+  asyncCheck t.connHandler(server, client)
 
 method init*(t: TcpTransport) =
   t.multicodec = multiCodec("tcp")

tests/testcrypto.nim

@@ -11,7 +11,7 @@
 ## https://github.com/libp2p/go-libp2p-crypto/blob/master/key.go
 import unittest
 import nimcrypto/[utils, sysrand]
-import ../libp2p/crypto/[crypto, chacha20poly1305, curve25519]
+import ../libp2p/crypto/[crypto, chacha20poly1305, curve25519, hkdf]
 
 when defined(nimHasUsed): {.used.}
 
@@ -480,6 +480,17 @@ suite "Key interface test suite":
     check ntag.toHex == tag.toHex
     ChaChaPoly.decrypt(key, nonce, ntag, text, aed)
     check text.toHex == plain.toHex
+    check ntag.toHex == tag.toHex
+
+    # ensure even a 2 byte array works
+    var
+      smallPlain: array[2, byte]
+      btag: ChaChaPolyTag
+      noaed: array[0, byte]
+    ChaChaPoly.encrypt(key, nonce, btag, smallPlain, noaed)
+    ntag = btag
+    ChaChaPoly.decrypt(key, nonce, btag, smallPlain, noaed)
+    check ntag.toHex == btag.toHex
 
   test "Curve25519":
     # from bearssl test_crypto.c
@@ -524,3 +535,26 @@ suite "Key interface test suite":
     check secret1.toHex == secret2.toHex
 
   
+  test "HKDF 1":
+    let
+      ikm = fromHex("0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b")
+      salt = fromHex("0x000102030405060708090a0b0c")
+      info = fromHex("0xf0f1f2f3f4f5f6f7f8f9")
+      truth = "3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf34007208d5b887185865"
+    var
+      output: array[1, array[42, byte]]
+
+    sha256.hkdf(salt, ikm, info, output)
+    check output[0].toHex(true) == truth
+
+  test "HKDF 2":
+    let
+      ikm = fromHex("0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b")
+      salt = fromHex("")
+      info = fromHex("")
+      truth = "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d9d201395faa4b61a96c8"
+    var
+      output: array[1, array[42, byte]]
+
+    sha256.hkdf(salt, ikm, info, output)
+    check output[0].toHex(true) == truth

tests/testnative.nim

@@ -7,6 +7,7 @@ import testtransport,
        testbufferstream,
        testidentify,
        testswitch,
+       testnoise,
        testpeerinfo,
        pubsub/testpubsub,
        # TODO: placing this before pubsub tests,

tests/testnoise.nim

@@ -0,0 +1,251 @@
+## Nim-LibP2P
+## Copyright (c) 2020 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import unittest, tables
+import chronos
+import chronicles
+import ../libp2p/crypto/crypto
+import ../libp2p/[switch,
+                  multistream,
+                  protocols/identify,
+                  connection,
+                  transports/transport,
+                  transports/tcptransport,
+                  multiaddress,
+                  peerinfo,
+                  crypto/crypto,
+                  peer,
+                  protocols/protocol,
+                  muxers/muxer,
+                  muxers/mplex/mplex,
+                  muxers/mplex/types,
+                  protocols/secure/noise,
+                  protocols/secure/secure]
+
+const TestCodec = "/test/proto/1.0.0"
+
+type
+  TestProto = ref object of LPProtocol
+
+method init(p: TestProto) {.gcsafe.} =
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    let msg = cast[string](await conn.readLp())
+    check "Hello!" == msg
+    await conn.writeLp("Hello!")
+    await conn.close()
+
+  p.codec = TestCodec
+  p.handler = handle
+
+proc createSwitch(ma: MultiAddress; outgoing: bool): (Switch, PeerInfo) =
+  var peerInfo: PeerInfo = PeerInfo.init(PrivateKey.random(RSA))
+  peerInfo.addrs.add(ma)
+  let identify = newIdentify(peerInfo)
+
+  proc createMplex(conn: Connection): Muxer =
+    result = newMplex(conn)
+
+  let mplexProvider = newMuxerProvider(createMplex, MplexCodec)
+  let transports = @[Transport(newTransport(TcpTransport))]
+  let muxers = [(MplexCodec, mplexProvider)].toTable()
+  let secureManagers = [(NoiseCodec, Secure(newNoise(peerInfo.privateKey, outgoing = outgoing)))].toTable()
+  let switch = newSwitch(peerInfo,
+                         transports,
+                         identify,
+                         muxers,
+                         secureManagers)
+  result = (switch, peerInfo)
+
+suite "Noise":
+  test "e2e: handle write + noise":
+    proc testListenerDialer(): Future[bool] {.async.} =
+      let
+        server: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
+        serverInfo = PeerInfo.init(PrivateKey.random(RSA), [server])
+        serverNoise = newNoise(serverInfo.privateKey, outgoing = false)
+
+      proc connHandler(conn: Connection) {.async, gcsafe.} =
+        let sconn = await serverNoise.secure(conn)
+        defer:
+          await sconn.close()
+        await sconn.write(cstring("Hello!"), 6)
+
+      let
+        transport1: TcpTransport = newTransport(TcpTransport)
+      asyncCheck await transport1.listen(server, connHandler)
+
+      let
+        transport2: TcpTransport = newTransport(TcpTransport)
+        clientInfo = PeerInfo.init(PrivateKey.random(RSA), [transport1.ma])
+        clientNoise = newNoise(clientInfo.privateKey, outgoing = true)
+        conn = await transport2.dial(transport1.ma)
+        sconn = await clientNoise.secure(conn)
+
+        msg = await sconn.read(6)
+
+      await sconn.close()
+      await transport1.close()
+
+      result = cast[string](msg) == "Hello!"
+
+    check:
+      waitFor(testListenerDialer()) == true
+
+  test "e2e: handle read + noise":
+    proc testListenerDialer(): Future[bool] {.async.} =
+      let
+        server: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
+        serverInfo = PeerInfo.init(PrivateKey.random(RSA), [server])
+        serverNoise = newNoise(serverInfo.privateKey, outgoing = false)
+        readTask = newFuture[void]()
+
+      proc connHandler(conn: Connection) {.async, gcsafe.} =
+        let sconn = await serverNoise.secure(conn)
+        defer:
+          await sconn.close()
+        let msg = await sconn.read(6)
+        check cast[string](msg) == "Hello!"
+        readTask.complete()
+
+      let
+        transport1: TcpTransport = newTransport(TcpTransport)
+      asyncCheck await transport1.listen(server, connHandler)
+
+      let
+        transport2: TcpTransport = newTransport(TcpTransport)
+        clientInfo = PeerInfo.init(PrivateKey.random(RSA), [transport1.ma])
+        clientNoise = newNoise(clientInfo.privateKey, outgoing = true)
+        conn = await transport2.dial(transport1.ma)
+        sconn = await clientNoise.secure(conn)
+
+      await sconn.write("Hello!".cstring, 6)
+      await readTask
+      await sconn.close()
+      await transport1.close()
+
+      result = true
+
+    check:
+      waitFor(testListenerDialer()) == true
+
+  test "e2e use switch dial proto string":
+    proc testSwitch(): Future[bool] {.async, gcsafe.} =
+      let ma1: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
+      let ma2: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
+
+      var peerInfo1, peerInfo2: PeerInfo
+      var switch1, switch2: Switch
+      var awaiters: seq[Future[void]]
+
+      (switch1, peerInfo1) = createSwitch(ma1, false)
+
+      let testProto = new TestProto
+      testProto.init()
+      testProto.codec = TestCodec
+      switch1.mount(testProto)
+      (switch2, peerInfo2) = createSwitch(ma2, true)
+      awaiters.add(await switch1.start())
+      awaiters.add(await switch2.start())
+      let conn = await switch2.dial(switch1.peerInfo, TestCodec)
+      await conn.writeLp("Hello!")
+      let msg = cast[string](await conn.readLp())
+      check "Hello!" == msg
+
+      await allFutures(switch1.stop(), switch2.stop())
+      await allFutures(awaiters)
+      result = true
+
+    check:
+      waitFor(testSwitch()) == true
+
+  # test "interop with rust noise":
+  #   when true: # disable cos in CI we got no interop server/client
+  #     proc testListenerDialer(): Future[bool] {.async.} =
+  #       const
+  #         proto = "/noise/xx/25519/chachapoly/sha256/0.1.0"
+
+  #       let
+  #         local = Multiaddress.init("/ip4/0.0.0.0/tcp/23456")
+  #         info = PeerInfo.init(PrivateKey.random(RSA), [local])
+  #         noise = newNoise(info.privateKey)
+  #         ms = newMultistream()
+  #         transport = TcpTransport.newTransport()
+
+  #       proc connHandler(conn: Connection) {.async, gcsafe.} =
+  #         try:
+  #           await ms.handle(conn)
+  #           trace "ms.handle exited"
+  #         except:
+  #           error getCurrentExceptionMsg()
+  #         finally:
+  #           await conn.close()
+     
+  #       ms.addHandler(proto, noise)
+
+  #       let
+  #         clientConn = await transport.listen(local, connHandler)
+  #       await clientConn
+
+  #       result = true
+
+  #     check:
+  #       waitFor(testListenerDialer()) == true
+
+  # test "interop with rust noise":
+  #   when true: # disable cos in CI we got no interop server/client
+  #     proc testListenerDialer(): Future[bool] {.async.} =
+  #       const
+  #         proto = "/noise/xx/25519/chachapoly/sha256/0.1.0"
+
+  #       let
+  #         local = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
+  #         remote = Multiaddress.init("/ip4/127.0.0.1/tcp/23456")
+  #         info = PeerInfo.init(PrivateKey.random(RSA), [local])
+  #         noise = newNoise(info.privateKey)
+  #         ms = newMultistream()
+  #         transport = TcpTransport.newTransport()
+  #         conn = await transport.dial(remote)
+
+  #       check ms.select(conn, @[proto]).await == proto
+
+  #       let
+  #         sconn = await noise.secure(conn, true)
+
+  #       # use sconn
+
+  #       result = true
+
+  #     check:
+  #       waitFor(testListenerDialer()) == true
+
+  # test "interop with go noise":
+  #   when true: # disable cos in CI we got no interop server/client
+  #     proc testListenerDialer(): Future[bool] {.async.} =
+  #       let
+  #         local = Multiaddress.init("/ip4/0.0.0.0/tcp/23456")
+  #         info = PeerInfo.init(PrivateKey.random(RSA), [local])
+  #         noise = newNoise(info.privateKey)
+  #         ms = newMultistream()
+  #         transport = TcpTransport.newTransport()
+
+  #       proc connHandler(conn: Connection) {.async, gcsafe.} =
+  #         try:
+  #           let seconn = await noise.secure(conn, false)
+  #           trace "ms.handle exited"
+  #         finally:
+  #           await conn.close()
+         
+  #       let
+  #         clientConn = await transport.listen(local, connHandler)
+  #       await clientConn
+
+  #       result = true
+
+  #     check:
+  #       waitFor(testListenerDialer()) == true