diff --git a/.gitignore b/.gitignore index 00420eb..ab53bdb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ master # Fuzzer-generated files crash-* +build/ diff --git a/snappy.nim b/snappy.nim index 79f39e3..9d7df1c 100644 --- a/snappy.nim +++ b/snappy.nim @@ -106,7 +106,7 @@ func uncompress*(input: openArray[byte], output: var openArray[byte]): if written.uint64 != lenU32: return err(CodecError.invalidInput) # Header does not match content - return ok(written) + ok(written) func decode*(input: openArray[byte], maxSize = maxUncompressedLen): seq[byte] = ## Decode input returning the uncompressed output. On error, return an empty @@ -117,7 +117,7 @@ func decode*(input: openArray[byte], maxSize = maxUncompressedLen): seq[byte] = let uncompressed = uncompressedLen(input).valueOr: return - if uncompressed > maxSize.uint64 or uncompressed > int.high.uint64: + if uncompressed > maxSize or uncompressed > int.high.uint64: return # TODO https://github.com/nim-lang/Nim/issues/19357 @@ -166,7 +166,8 @@ func encodeFramed*(input: openArray[byte]): seq[byte] = result.setLen(written) func uncompressFramed*( - input: openArray[byte], output: var openArray[byte], checkHeader = true): + input: openArray[byte], output: var openArray[byte], checkHeader = true, + checkIntegrity = true): Result[tuple[read: int, written: int], FrameError] = ## Uncompress as many frames as possible from `input` and write them to ## `output`, returning the number of bytes read and written. @@ -226,7 +227,7 @@ func uncompressFramed*( of CodecError.invalidInput: err(FrameError.invalidInput) return res - if maskedCrc( + if checkIntegrity and maskedCrc( output.toOpenArray(written, written + (uncompressed - 1))) != crc: return err(FrameError.crcMismatch) @@ -239,7 +240,8 @@ func uncompressFramed*( let crc = uint32.fromBytesLE input.toOpenArray(read, read + 3) - if maskedCrc(input.toOpenArray(read + 4, read + (dataLen - 1))) != crc: + if checkIntegrity and + maskedCrc(input.toOpenArray(read + 4, read + (dataLen - 1))) != crc: return err(FrameError.crcMismatch) let uncompressed = dataLen - 4 # dataLen includes CRC length @@ -263,7 +265,9 @@ func uncompressFramed*( ok((read, written)) -func decodeFramed*(input: openArray[byte], maxSize = int.high): seq[byte] = +func decodeFramed*( + input: openArray[byte], maxSize = int.high, + checkIntegrity = true): seq[byte] = ## Uncompress as many frames as possible from `input` and return the ## uncompressed output. ## @@ -275,13 +279,13 @@ func decodeFramed*(input: openArray[byte], maxSize = int.high): seq[byte] = let uncompressed = uncompressedLenFramed(input).valueOr: return - if uncompressed > maxSize.uint64 or uncompressed > int.high.uint64: + if uncompressed > maxSize.uint64: return # TODO https://github.com/nim-lang/Nim/issues/19357 result = newSeqUninitialized[byte](int uncompressed) - if uncompressFramed(input, result).isErr(): + if uncompressFramed(input, result, checkIntegrity = checkIntegrity).isErr(): result = @[] # Empty return on error template compress*(input: openArray[byte]): seq[byte] {. diff --git a/snappy/codec.nim b/snappy/codec.nim index 72a872d..5c541bb 100644 --- a/snappy/codec.nim +++ b/snappy/codec.nim @@ -196,7 +196,7 @@ func uncompressedLenFramed*(input: openArray[byte]): Opt[uint64] = else: 0'u32 # Reserved skippable (for example framing format header) if uncompressed > maxUncompressedFrameDataLen: - return # Uncomnpressed data has limits (for the known chunk types) + return # Uncompressed data has limits (for the known chunk types) expected += uncompressed read += dataLen diff --git a/snappy/faststreams.nim b/snappy/faststreams.nim index 74d4053..89a2ee4 100644 --- a/snappy/faststreams.nim +++ b/snappy/faststreams.nim @@ -10,9 +10,10 @@ export inputs, multisync, outputs, codec, exceptions proc checkCrcAndAppend( - output: OutputStream, data: openArray[byte], crc: uint32): bool {. + output: OutputStream, data: openArray[byte], crc: uint32, + checkIntegrity: bool): bool {. raises: [IOError].}= - if maskedCrc(data) == crc: + if not checkIntegrity or maskedCrc(data) == crc: output.write(data) return true @@ -85,7 +86,8 @@ proc compressFramed*(input: openArray[byte], output: OutputStream) {. raises: [IOError].} = compressFramed(unsafeMemoryInput(input), output) -proc uncompressFramed*(input: InputStream, output: OutputStream) {. +proc uncompressFramed*( + input: InputStream, output: OutputStream, checkIntegrity = true) {. fsMultiSync, raises: [IOError, SnappyDecodingError].} = if not input.readable(framingHeader.len): raise newException(UnexpectedEofError, "Failed to read stream header") @@ -112,7 +114,8 @@ proc uncompressFramed*(input: InputStream, output: OutputStream) {. uncompressed = uncompress(input.read(dataLen - 4), tmp).valueOr: raise newException(MalformedSnappyData, "Failed to decompress content") - if not checkCrcAndAppend(Sync output, tmp.toOpenArray(0, uncompressed-1), crc): + if not checkCrcAndAppend( + Sync output, tmp.toOpenArray(0, uncompressed-1), crc, checkIntegrity): raise newException(MalformedSnappyData, "Content CRC checksum failed") elif id == chunkUncompressed: @@ -123,7 +126,8 @@ proc uncompressFramed*(input: InputStream, output: OutputStream) {. raise newException(MalformedSnappyData, "Invalid frame length: " & $dataLen) let crc = uint32.fromBytesLE(input.read(4)) - if not checkCrcAndAppend(Sync output, input.read(dataLen - 4), crc): + if not checkCrcAndAppend( + Sync output, input.read(dataLen - 4), crc, checkIntegrity): raise newException(MalformedSnappyData, "Content CRC checksum failed") elif id < 0x80: @@ -142,6 +146,8 @@ proc uncompressFramed*(input: InputStream, output: OutputStream) {. output.flush() -proc uncompressFramed*(input: openArray[byte], output: OutputStream) {. +proc uncompressFramed*( + input: openArray[byte], output: OutputStream, checkIntegrity = true) {. raises: [IOError, SnappyDecodingError].} = - uncompressFramed(unsafeMemoryInput(input), output) + uncompressFramed( + unsafeMemoryInput(input), output, checkIntegrity = checkIntegrity) diff --git a/tests/benchmark.nim b/tests/benchmark.nim index e06924f..9f35780 100644 --- a/tests/benchmark.nim +++ b/tests/benchmark.nim @@ -49,7 +49,13 @@ proc readSource(sourceName: string): seq[byte] = doAssert(size == f.readBytes(result, 0, size)) f.close() -proc streamsEncode(input: openArray[byte]): seq[byte] = +proc memEncode(input: openArray[byte]): seq[byte] {.noinline.} = + snappy.encode(input) + +proc memDecode(input: openArray[byte]): seq[byte] {.noinline.} = + snappy.decode(input) + +proc streamsEncode(input: openArray[byte]): seq[byte] {.noinline.} = let ins = newStringStream(string.fromBytes(input)) outs = newStringStream() @@ -57,21 +63,27 @@ proc streamsEncode(input: openArray[byte]): seq[byte] = outs.setPosition(0) outs.readAll().toBytes() # This line is a hotspot due to missing RVO -proc faststreamsEncode(input: openArray[byte]): seq[byte] = +proc faststreamsEncode(input: openArray[byte]): seq[byte] {.noinline.} = let ins = unsafeMemoryInput(input) outs = memoryOutput() compress(ins, outs) outs.getOutput() # This line is a hotspot due to missing RVO -proc faststreamsEncodeFramed(input: openArray[byte]): seq[byte] = +proc memEncodeFramed(input: openArray[byte]): seq[byte] {.noinline.} = + snappy.encodeFramed(input) + +proc memDecodeFramed(input: openArray[byte]): seq[byte] {.noinline.} = + snappy.decodeFramed(input) + +proc faststreamsEncodeFramed(input: openArray[byte]): seq[byte] {.noinline.} = let ins = unsafeMemoryInput(input) outs = memoryOutput() compressFramed(ins, outs) outs.getOutput() # This line is a hotspot due to missing RVO -proc faststreamsDecodeFramed(input: openArray[byte]): seq[byte] = +proc faststreamsDecodeFramed(input: openArray[byte]): seq[byte] {.noinline.} = let ins = unsafeMemoryInput(input) outs = memoryOutput() @@ -87,9 +99,9 @@ proc timedRoundTrip(msg: string, source: openArray[byte], iterations = 100) = for i in 0..