status-im
/
nim-eth-p2p
mirror of https://github.com/status-im/nim-eth-p2p.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

whisper: add primitives (WIP) 

Work-in-progress dump of initial whisper support

* payload encoding / decoding (missing 256-bit GCM mode)
* missing tests
* missing p2p integration
This commit is contained in:
Jacek Sieka 2018-09-26 15:08:24 -06:00 committed by Zahary Karadjov
parent 7828ef0481
commit 7fdd574a37
1 changed files with 393 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

393
eth_p2p/rlpx_protocols/shh.nim Normal file
View File

@ -0,0 +1,393 @@
## Whisper
##
## Whisper is a gossip protocol that synchronizes a set of messages across nodes
## with attention given to sender and recipient anonymitiy. Messages are
## categorized by a topic and stay alive in the network based on a time-to-live
## measured in seconds. Spam prevention is based on proof-of-work, where large
## or long-lived messages must spend more work.
import
algorithm,
bitops,
endians,
eth_keys,
eth_p2p/ecies,
math,
nimcrypto/hash,
nimcrypto/keccak,
nimcrypto/rijndael,
options,
rlp/types,
rlp/writer,
secp256k1,
sequtils,
strutils,
tables,
times
const
PadLengthMask = 0b11000000'u8
PadLengthPos = 6
SignedMask = 0b00100000'u8
type
Hash = MDigest[256]
SymKey = array[256 div 8, byte] ## AES256 key
Topic = array[4, byte]
Bloom = array[64, byte] ## XXX: nim-eth-bloom has really quirky API and fixed
## bloom size.
## stint is massive overkill / poor fit - a bloom filter is an array of bits,
## not a number
Payload = object
## Payload is what goes in the data field of the Envelope
src: Option[PrivateKey] ## Optional key used for signing message
dst: Option[PublicKey] ## Optional key used for asymmetric encryption
symKey: Option[SymKey] ## Optional key used for symmetric encryption
payload: seq[byte] ## Application data / message contents
padding: seq[byte] ## Padding - if empty, will automatically pad up to
## nearest 256-byte boundary
Envelope = object
## What goes on the wire in the whisper protocol - a payload and some
## book-keeping
## Don't touch field order, there's lots of macro magic that depends on it
expiry: uint64 ## Unix timestamp when message expires
ttl: uint64 ## Time-to-live, seconds - message was created at (expiry - ttl)
topic: Topic
data: seq[byte] ## Payload, as given by user
nonce: uint64 ## Nonce used for proof-of-work calculation
Message = object
## An Envelope with a few cached properties
env: Envelope
hash: Hash ## Hash, as calculated for proof-of-work
size: uint64 ## RLP-encoded size of message
pow: float64 ## Calculated proof-of-work
bloom: Bloom ## Filter sent to direct peers for topic-based filtering
Queue = object
## Bounded message repository
##
## Whisper uses proof-of-work to judge the usefulness of a message staying
## in the "cloud" - messages with low proof-of-work will be removed to make
## room for those with higher pow, even if they haven't expired yet.
## Larger messages and those with high time-to-live will require more pow.
items: seq[Message] ## Sorted by proof-of-work
capacity: int ## Max messages to keep. \
## XXX: really big messages can cause excessive mem usage when using msg \
## count
# Utilities --------------------------------------------------------------------
proc toBE(v: uint64): array[8, byte] =
# return uint64 as bigendian array - for easy consumption with hash function
var v = cast[array[8, byte]](v)
bigEndian64(result.addr, v.addr)
proc toLE(v: uint32): array[4, byte] =
# return uint32 as bigendian array - for easy consumption with hash function
var v = cast[array[4, byte]](v)
littleEndian32(result.addr, v.addr)
# XXX: get rid of pointer
proc fromLE32(v: array[4, byte]): uint32 =
var v = v
var ret: array[4, byte]
littleEndian32(ret.addr, v.addr)
result = cast[uint32](ret)
proc leadingZeroBits(hash: MDigest): int =
## Number of most significant zero bits before the first one
for h in hash.data:
static: assert sizeof(h) == 1
if h == 0:
result += 8
else:
result += countLeadingZeroBits(h)
break
proc calcPow(size, ttl: uint64, hash: Hash): float64 =
## Whisper proof-of-work is defined as the best bit of a hash divided by
## encoded size and time-to-live, such that large and long-lived messages get
## penalized
let bits = leadingZeroBits(hash) + 1
return pow(2.0, bits.float64) / (size.float64 * ttl.float64)
proc topicBloom(topic: Topic): Bloom =
## Whisper uses 512-bit bloom filters meaning 9 bits of indexing - 3 9-bit
## indexes into the bloom are created using the first 3 bytes of the topic and
## complementing each byte with an extra bit from the last topic byte
for i in 0..<3:
var idx = uint16(topic[i])
if (topic[3] and byte(1 shl i)) != 0: # fetch the 9'th bit from the last byte
idx = idx + 256
assert idx <= 511
result[idx div 8] = result[idx div 8] or byte(1 shl (idx and 7'u16))
# Payloads ---------------------------------------------------------------------
# Several differences between geth and parity - this code is closer to geth
# simply because that makes it closer to EIP 627 - see also:
# https://github.com/paritytech/parity-ethereum/issues/9652
proc encode*(self: Payload): seq[byte] =
## Encode a payload according so as to make it suitable to put in an Envelope
const
FlagsLen = 1
PadMaxLen = 256
# length of the payload length field :)
# XXX: deal with those extra large inputs we can't send
let payloadLenLen =
if self.payload.len >= 256*256: 3'u8
elif self.payload.len >= 256: 2'u8
else: 1'u8
let signatureLen =
if self.src.isSome(): RawSignatureSize
else: 0
# Upper boundary for buffer needs - we'll likely use a bit less
let maxLen = FlagsLen + payloadLenLen.int + self.payload.len +
self.padding.len + signatureLen + PadMaxLen
var plain = newSeqOfCap[byte](maxLen)
let signatureFlag =
if self.src.isSome(): 0b100'u8
else: 0'u8
# byte 0: flags with payload length length and presence of signature
plain.add payloadLenLen or signatureFlag
# next, length of payload - little endian (who comes up with this stuff? why
# can't the world just settle on one endian?)
let payloadLen = self.payload.len.uint32.toLE
# No, I have no love for nim closed ranges - such a mess to remember the extra
# < or risk off-by-ones when working with lengths..
plain.add payloadLen[0..<payloadLenLen]
plain.add self.payload
if self.padding.len > 0:
plain.add self.padding
else:
let len = FlagsLen + payloadLenLen.int + self.payload.len + signatureLen
let padLen = (len + 255) mod 256
plain.add repeat(0'u8, padLen) # XXX: should be random
if self.src.isSome(): # Private key present - signature requested
let hash = keccak256.digest(plain)
var sig: Signature
# XXX: ugh, this raises sometimes, and returns a status code.. lovely.
# XXX: handle some errors, or something
discard signRawMessage(hash.data, self.src.get(), sig)
plain.add sig.getRaw()
if self.dst.isSome(): # Asymmetric key present - encryption requested
result.setLen eciesEncryptedLength(plain.len)
# XXX: handle those errors here also
discard eciesEncrypt(plain, result, self.dst.get())
elif self.symKey.isSome(): # Symmetric key present - encryption requested
# https://github.com/cheatfate/nimcrypto/issues/11
assert false, "no 256-bit GCM support in nimcrypto"
else: # No encryption!
result = plain
proc decode*(self: var Payload, data: openarray[byte]): bool =
## Decode data into payload, using keys found in self
var plain: seq[byte]
if self.src.isSome():
plain.setLen(eciesDecryptedLength(data.len))
if eciesDecrypt(data, plain, self.src.get()) != EciesStatus.Success:
return false
elif self.symKey.isSome():
# https://github.com/cheatfate/nimcrypto/issues/11
assert false, "no 256-bit GCM support in nimcrypto"
else: # No encryption!
plain = @data
# XXX: bounds checking??
let payloadLenLen = plain[0] and 0b11'u8
let hasSignature = (plain[0] and 0b100'u8) != 0
var payloadLen32: array[4, byte]
for i in 0..<payloadLenLen.int: payloadLen32[i] = data[1 + i]
let payloadLen = payloadLen32.fromLE32()
self.payload.add data[2..<payloadLen + 2]
# XXX check signatures and stuff..
# Envelopes --------------------------------------------------------------------
proc valid*(self: Envelope, now = epochTime()): bool =
if self.expiry.float64 < now: return false # expired
if self.ttl <= 0: return false # this would invalidate pow calculation
let created = self.expiry - self.ttl
if created.float64 > (now + 2.0): return false # created in the future
return true
proc toShortRlp(self: Envelope): seq[byte] =
## RLP-encoded message without nonce is used during proof-of-work calculations
writer.encodeList(self.expiry, self.ttl, self.topic, self.data)
proc toRlp(self: Envelope): seq[byte] =
## What gets sent out over the wire includes the nonce
writer.encode(self)
proc minePow*(self: Envelope, seconds: float): uint64 =
## For the given envelope, spend millis milliseconds to find the
## best proof-of-work and return the nonce
let bytes = self.toShortRlp()
var ctx: keccak256
ctx.init()
ctx.update(bytes)
var bestPow: float64 = 0.0
let mineEnd = epochTime() + seconds
var i: uint64
while epochTime() < mineEnd or bestPow == 0: # At least one round
var tmp = ctx # copy hash calculated so far - we'll reuse that for each iter
tmp.update(i.toBE())
i.inc
# XXX:a random nonce here would not leak number of iters
let pow = calcPow(1, 1, tmp.finish())
if pow > bestPow: # XXX: could also compare hashes as numbers instead
bestPow = pow
result = i.uint64
proc calcPowHash(self: Envelope): Hash =
## Calculate the message hash, as done during mining - this can be used to
## verify proof-of-work
let bytes = self.toShortRlp()
var ctx: keccak256
ctx.init()
ctx.update(bytes)
ctx.update(self.nonce.toBE())
return ctx.finish()
# Messages ---------------------------------------------------------------------
proc cmpPow(a, b: Message): int =
## Biggest pow first, lowest at the end (for easy popping)
if a.pow > b.pow: 1
elif a.pow == b.pow: 0
else: -1
proc initMessage(env: Envelope): Message =
result.env = env
result.hash = env.calcPowHash()
result.size = env.toRlp().len().uint64 # XXX: calc len without creating RLP
result.pow = calcPow(result.size, result.env.ttl, result.hash)
# Queues -----------------------------------------------------------------------
proc initQueue(capacity: int): Queue =
result.items = newSeqOfCap[Message](capacity)
result.capacity = capacity
proc prune(self: var Queue) =
## Remove items that are past their expiry time
let now = epochTime().uint64
self.items.keepIf(proc(m: Message): bool = m.env.expiry > now)
proc add(self: var Queue, msg: Message) =
## Add a message to the queue.
## If we're at capacity, we will be removing, in order:
## * expired messages
## * lowest proof-of-work message - this may be `msg` itself!
if self.items.len >= self.capacity:
self.prune() # Only prune if needed
if self.items.len >= self.capacity:
# Still no room - go by proof-of-work quantity
let last = self.items[^1]
if last.pow > msg.pow or
(last.pow == msg.pow and last.env.expiry > msg.env.expiry):
# The new message has less pow or will expire earlier - drop it
self.items.del(self.items.len() - 1)
self.items.insert(msg, self.items.lowerBound(msg, cmpPow))
when false:
rlpxProtocol shh, 6:
proc status(p: Peer, values: openarray[KeyValuePair]) =
discard
proc status(p: Peer, values: openarray[KeyValuePair]) =
discard
proc messages(p: Peer, values: openarray[KeyValuePair]) =
discard
proc powRequirement(p: Peer, values: openarray[KeyValuePair]) =
discard
proc topicFilter(p: Peer, values: openarray[KeyValuePair]) =
discard
if isMainModule:
block:
# Geth test: https://github.com/ethersphere/go-ethereum/blob/d3441ebb563439bac0837d70591f92e2c6080303/whisper/whisperv6/whisper_test.go#L834
let top0 = [byte 0, 0, 255, 6]
var x: Bloom
x[0] = byte 1
x[32] = byte 1
x[^1] = byte 128
doAssert @(top0.topicBloom) == @x
# example from https://github.com/paritytech/parity-ethereum/blob/93e1040d07e385d1219d00af71c46c720b0a1acf/whisper/src/message.rs#L439
let
env0 = Envelope(expiry:100000, ttl: 30, topic: [byte 0, 0, 0, 0], data: repeat(byte 9, 256), nonce: 1010101)
env1 = Envelope(expiry:100000, ttl: 30, topic: [byte 0, 0, 0, 0], data: repeat(byte 9, 256), nonce: 1010102)
block:
# XXX checked with parity, should check with geth too - found a potential bug
# in parity while playing with it:
# https://github.com/paritytech/parity-ethereum/issues/9625
doAssert $calcPowHash(env0) == "A13B48480AEB3123CD2358516E2E8EE9FCB0F4CB37E68CD09FDF7F9A7E14767C"
block:
var queue = initQueue(1)
let msg0 = initMessage(env0)
let msg1 = initMessage(env1)
queue.add(msg0)
queue.add(msg1)
doAssert queue.items.len() == 1
doAssert queue.items[0].env.nonce ==
(if msg0.pow > msg1.pow: msg0.env.nonce else: msg1.env.nonce)
block:
var queue = initQueue(2)
queue.add(initMessage(env0))
queue.add(initMessage(env1))
doAssert queue.items.len() == 2
block:
doAssert writer.encode(env0) ==
writer.encodeList(env0.expiry, env0.ttl, env0.topic, env0.data, env0.nonce)