use stew/leb128
* 3 varint parsers -> 1 * 1/3 memory usage / allocations when writing stuff to libp2p
This commit is contained in:
parent
9211b1ca03
commit
4e191a06ac
|
@ -4,15 +4,14 @@ import
|
||||||
std/options as stdOptions,
|
std/options as stdOptions,
|
||||||
|
|
||||||
# Status libs
|
# Status libs
|
||||||
stew/[varints, base58, endians2, results, byteutils, io2], bearssl,
|
stew/[leb128, base58, endians2, results, byteutils, io2], bearssl,
|
||||||
stew/shims/net as stewNet,
|
stew/shims/net as stewNet,
|
||||||
stew/shims/[macros, tables],
|
stew/shims/[macros, tables],
|
||||||
faststreams/[inputs, outputs, buffers], snappy, snappy/framing,
|
faststreams/[inputs, outputs, buffers], snappy, snappy/framing,
|
||||||
json_serialization, json_serialization/std/[net, options],
|
json_serialization, json_serialization/std/[net, options],
|
||||||
chronos, chronicles, metrics,
|
chronos, chronicles, metrics,
|
||||||
# TODO: create simpler to use libp2p modules that use re-exports
|
|
||||||
libp2p/[switch, peerinfo,
|
libp2p/[switch, peerinfo,
|
||||||
multiaddress, multicodec, crypto/crypto, crypto/secp,
|
multiaddress, crypto/crypto, crypto/secp,
|
||||||
protocols/identify, protocols/protocol],
|
protocols/identify, protocols/protocol],
|
||||||
libp2p/muxers/muxer, libp2p/muxers/mplex/mplex,
|
libp2p/muxers/muxer, libp2p/muxers/mplex/mplex,
|
||||||
libp2p/transports/[transport, tcptransport],
|
libp2p/transports/[transport, tcptransport],
|
||||||
|
@ -496,8 +495,8 @@ proc writeChunk*(conn: Connection,
|
||||||
if responseCode.isSome:
|
if responseCode.isSome:
|
||||||
output.write byte(responseCode.get)
|
output.write byte(responseCode.get)
|
||||||
|
|
||||||
output.write varintBytes(payload.lenu64)
|
output.write toBytes(payload.lenu64, Leb128).toOpenArray()
|
||||||
output.write(framingFormatCompress payload)
|
framingFormatCompress(output, payload)
|
||||||
|
|
||||||
conn.write(output.getOutput)
|
conn.write(output.getOutput)
|
||||||
|
|
||||||
|
@ -505,14 +504,14 @@ template errorMsgLit(x: static string): ErrorMsg =
|
||||||
const val = ErrorMsg toBytes(x)
|
const val = ErrorMsg toBytes(x)
|
||||||
val
|
val
|
||||||
|
|
||||||
func formatErrorMsg(msg: ErrorMSg): string =
|
func formatErrorMsg(msg: ErrorMsg): string =
|
||||||
let candidate = string.fromBytes(asSeq(msg))
|
# ErrorMsg "usually" contains a human-readable string - we'll try to parse it
|
||||||
for c in candidate:
|
# as ASCII and return hex if that fails
|
||||||
# TODO UTF-8 - but let's start with ASCII
|
for c in msg:
|
||||||
if ord(c) < 32 or ord(c) > 127:
|
if c < 32 or c > 127:
|
||||||
return byteutils.toHex(asSeq(msg))
|
return byteutils.toHex(asSeq(msg))
|
||||||
|
|
||||||
return candidate
|
string.fromBytes(asSeq(msg))
|
||||||
|
|
||||||
proc sendErrorResponse(peer: Peer,
|
proc sendErrorResponse(peer: Peer,
|
||||||
conn: Connection,
|
conn: Connection,
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
|
|
||||||
import
|
import
|
||||||
std/[typetraits, options],
|
std/[typetraits, options],
|
||||||
stew/[bitops2, endians2, objects],
|
stew/[bitops2, endians2, leb128, objects],
|
||||||
serialization, serialization/testing/tracing,
|
serialization, serialization/testing/tracing,
|
||||||
../spec/[digest, datatypes],
|
../spec/[digest, datatypes],
|
||||||
./bytes_reader, ./bitseqs, ./types, ./spec_types
|
./bytes_reader, ./bitseqs, ./types, ./spec_types
|
||||||
|
@ -219,18 +219,11 @@ func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} =
|
||||||
unsupported T
|
unsupported T
|
||||||
|
|
||||||
proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} =
|
proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} =
|
||||||
var cursor = w.stream.delayVarSizeWrite(10)
|
var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64))
|
||||||
let initPos = w.stream.pos
|
let initPos = w.stream.pos
|
||||||
w.writeValue T(x)
|
w.writeValue T(x)
|
||||||
let length = uint64(w.stream.pos - initPos)
|
let length = toBytes(uint64(w.stream.pos - initPos), Leb128)
|
||||||
when false:
|
cursor.finalWrite length.toOpenArray()
|
||||||
discard
|
|
||||||
# TODO varintBytes is sub-optimal at the moment
|
|
||||||
# cursor.writeAndFinalize length.varintBytes
|
|
||||||
else:
|
|
||||||
var buf: VarintBuffer
|
|
||||||
buf.writeVarint length
|
|
||||||
cursor.finalWrite buf.writtenBytes
|
|
||||||
|
|
||||||
proc readValue*[T](r: var SszReader, val: var T) {.raises: [Defect, MalformedSszError, SszSizeMismatchError, IOError].} =
|
proc readValue*[T](r: var SszReader, val: var T) {.raises: [Defect, MalformedSszError, SszSizeMismatchError, IOError].} =
|
||||||
when isFixedSize(T):
|
when isFixedSize(T):
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
Subproject commit 1befeb8c2e74425787d7bfd7b53d6c60302161c7
|
Subproject commit 2f90645669d18626f928300b918dc14664e00051
|
|
@ -1 +1 @@
|
||||||
Subproject commit 8455b825e50be706eb689f7629dac7098d2d5da5
|
Subproject commit b2f48c0fd331a72fc917cd9cd7fadea683ca9de2
|
|
@ -1 +1 @@
|
||||||
Subproject commit e15c1ae01269b1c53e1c9d80741b6d8929fc4b75
|
Subproject commit 53979b7c5a5d18b015b0e5296d7d02c72e92eb5a
|
Loading…
Reference in New Issue