diff --git a/src/zippy/deflate.nim b/src/zippy/deflate.nim index 38428bc..f629754 100644 --- a/src/zippy/deflate.nim +++ b/src/zippy/deflate.nim @@ -348,164 +348,165 @@ func deflate*(src: seq[uint8]): seq[uint8] = if len > 0: b.addBytes(src[pos].unsafeAddr, len.int) - b.data.setLen(b.data.len) - else: - # Deflate using dynamic Huffman tree + b.data.setLen(b.bytePos) + return b.data - let - (llNumCodes, llLengths, llCodes) = huffmanCodeLengths( - freqLitLen, 257, maxLitLenCodeLength - ) - (distNumCodes, distLengths, distCodes) = huffmanCodeLengths( - freqDist, 2, maxDistCodeLength - ) + # Deflate using dynamic Huffman tree - var bitLens = newSeqOfCap[uint8](llNumCodes + distNumCodes) - for i in 0 ..< llNumCodes: - bitLens.add(llLengths[i]) - for i in 0 ..< distNumCodes: - bitLens.add(distLengths[i]) - - var - bitLensRle: seq[uint8] - bitCount: int - block construct_binlens_rle: - var i: int - while i < bitLens.len: - var repeatCount: int - while i + repeatCount + 1 < bitLens.len and - bitLens[i + repeatCount + 1] == bitLens[i]: - inc repeatCount - - if bitLens[i] == 0 and repeatCount >= 2: - inc repeatCount # Initial zero - if repeatCount <= 10: - bitLensRle.add([17.uint8, repeatCount.uint8 - 3]) - else: - repeatCount = min(repeatCount, 138) # Max of 138 zeros for code 18 - bitLensRle.add([18.uint8, repeatCount.uint8 - 11]) - inc(i, repeatCount - 1) - inc(bitCount, 7) - elif repeatCount >= 3: # Repeat code for non-zero, must be >= 3 times - var - a = repeatCount div 6 - b = repeatCount mod 6 - bitLensRle.add(bitLens[i]) - for j in 0 ..< a: - bitLensRle.add([16.uint8, 3]) - if b >= 3: - bitLensRle.add([16.uint8, b.uint8 - 3]) - else: - dec(repeatCount, b) - inc(i, repeatCount) - inc(bitCount, (a + b) * 2) - else: - bitLensRle.add(bitLens[i]) - inc i - inc(bitCount, 7) - - var clFreq = newSeq[int](19) - block count_cl_frequencies: - var i: int - while i < bitLensRle.len: - inc clFreq[bitLensRle[i]] - # Skip the number of times codes are repeated - if bitLensRle[i] >= 16: - inc i - inc i - - let (_, clLengths, clCodes) = huffmanCodeLengths(clFreq, clFreq.len, 7) - - var bitLensCodeLen = newSeq[uint8](clFreq.len) - for i in 0 ..< bitLensCodeLen.len: - bitLensCodeLen[i] = clLengths[clclOrder[i]] - - while bitLensCodeLen[bitLensCodeLen.high] == 0 and bitLensCodeLen.len > 4: - bitLensCodeLen.setLen(bitLensCodeLen.len - 1) - - let - hlit = (llNumCodes - 257).uint8 - hdist = distNumCodes.uint8 - 1 - hclen = bitLensCodeLen.len.uint8 - 4 - - # TODO: Improve the b.data.setLens - b.data.setLen( - b.data.len + - (((hclen.int + 4) * 3 + 7) div 8) + # hclen rle - bitLensRle.len * 2 + let + (llNumCodes, llLengths, llCodes) = huffmanCodeLengths( + freqLitLen, 257, maxLitLenCodeLength + ) + (distNumCodes, distLengths, distCodes) = huffmanCodeLengths( + freqDist, 2, maxDistCodeLength ) - b.addBit(1) - b.addBits(2, 2) + var bitLens = newSeqOfCap[uint8](llNumCodes + distNumCodes) + for i in 0 ..< llNumCodes: + bitLens.add(llLengths[i]) + for i in 0 ..< distNumCodes: + bitLens.add(distLengths[i]) - b.addBits(hlit, 5) - b.addBits(hdist, 5) - b.addBits(hclen, 4) + var + bitLensRle: seq[uint8] + bitCount: int + block construct_binlens_rle: + var i: int + while i < bitLens.len: + var repeatCount: int + while i + repeatCount + 1 < bitLens.len and + bitLens[i + repeatCount + 1] == bitLens[i]: + inc repeatCount - for i in 0.uint8 ..< hclen + 4: - b.addBits(bitLensCodeLen[i], 3) - - block write_bitlens_rle: - var i: int - while i < bitLensRle.len: - let symbol = bitLensRle[i] - b.addBits(clCodes[symbol], clLengths[symbol].int) - inc i - if symbol == 16: - b.addBits(bitLensRle[i], 2) - inc i - elif symbol == 17: - b.addBits(bitLensRle[i], 3) - inc i - elif symbol == 18: - b.addBits(bitLensRle[i], 7) - inc i - - block write_encoded_data: - var srcPos, encPos: int - while encPos < encoded.len: - if (encoded[encPos] and (1 shl 15)) != 0: - let - value = encoded[encPos] - offset = encoded[encPos + 1] - length = encoded[encPos + 2] - lengthIndex = (value shr 8) and (uint8.high shr 1) - distIndex = value and uint8.high - lengthExtraBits = baseLengthsExtraBits[lengthIndex] - lengthExtra = length - baseLengths[lengthIndex] - distExtraBits = baseDistanceExtraBits[distIndex] - distExtra = offset - baseDistance[distIndex] - inc(encPos, 3) - inc(srcPos, length.int) - - if b.data.len < b.bytePos + 6: - b.data.setLen(b.data.len * 2) - - b.addBits( - llCodes[lengthIndex + firstLengthCodeIndex], - llLengths[lengthIndex + firstLengthCodeIndex].int - ) - b.addBits(lengthExtra, lengthExtraBits) - b.addBits(distCodes[distIndex], distLengths[distIndex].int) - b.addBits(distExtra, distExtraBits) + if bitLens[i] == 0 and repeatCount >= 2: + inc repeatCount # Initial zero + if repeatCount <= 10: + bitLensRle.add([17.uint8, repeatCount.uint8 - 3]) else: - let length = encoded[encPos].int - inc encPos + repeatCount = min(repeatCount, 138) # Max of 138 zeros for code 18 + bitLensRle.add([18.uint8, repeatCount.uint8 - 11]) + inc(i, repeatCount - 1) + inc(bitCount, 7) + elif repeatCount >= 3: # Repeat code for non-zero, must be >= 3 times + var + a = repeatCount div 6 + b = repeatCount mod 6 + bitLensRle.add(bitLens[i]) + for j in 0 ..< a: + bitLensRle.add([16.uint8, 3]) + if b >= 3: + bitLensRle.add([16.uint8, b.uint8 - 3]) + else: + dec(repeatCount, b) + inc(i, repeatCount) + inc(bitCount, (a + b) * 2) + else: + bitLensRle.add(bitLens[i]) + inc i + inc(bitCount, 7) - let worstCaseBytesNeeded = (length * maxLitLenCodeLength + 7) div 8 - if b.data.len < b.bytePos + worstCaseBytesNeeded: - b.data.setLen(max(b.bytePos + worstCaseBytesNeeded, b.data.len * 2)) + var clFreq = newSeq[int](19) + block count_cl_frequencies: + var i: int + while i < bitLensRle.len: + inc clFreq[bitLensRle[i]] + # Skip the number of times codes are repeated + if bitLensRle[i] >= 16: + inc i + inc i - for j in 0 ..< length: - b.addBits(llCodes[src[srcPos]], llLengths[src[srcPos]].int) - inc srcPos + let (_, clLengths, clCodes) = huffmanCodeLengths(clFreq, clFreq.len, 7) - if llLengths[256] == 0: - failCompress() + var bitLensCodeLen = newSeq[uint8](clFreq.len) + for i in 0 ..< bitLensCodeLen.len: + bitLensCodeLen[i] = clLengths[clclOrder[i]] - b.addBits(llCodes[256], llLengths[256].int) # End of block + while bitLensCodeLen[bitLensCodeLen.high] == 0 and bitLensCodeLen.len > 4: + bitLensCodeLen.setLen(bitLensCodeLen.len - 1) - b.skipRemainingBitsInCurrentByte() + let + hlit = (llNumCodes - 257).uint8 + hdist = distNumCodes.uint8 - 1 + hclen = bitLensCodeLen.len.uint8 - 4 + + # TODO: Improve the b.data.setLens + b.data.setLen( + b.data.len + + (((hclen.int + 4) * 3 + 7) div 8) + # hclen rle + bitLensRle.len * 2 + ) + + b.addBit(1) + b.addBits(2, 2) + + b.addBits(hlit, 5) + b.addBits(hdist, 5) + b.addBits(hclen, 4) + + for i in 0.uint8 ..< hclen + 4: + b.addBits(bitLensCodeLen[i], 3) + + block write_bitlens_rle: + var i: int + while i < bitLensRle.len: + let symbol = bitLensRle[i] + b.addBits(clCodes[symbol], clLengths[symbol].int) + inc i + if symbol == 16: + b.addBits(bitLensRle[i], 2) + inc i + elif symbol == 17: + b.addBits(bitLensRle[i], 3) + inc i + elif symbol == 18: + b.addBits(bitLensRle[i], 7) + inc i + + block write_encoded_data: + var srcPos, encPos: int + while encPos < encoded.len: + if (encoded[encPos] and (1 shl 15)) != 0: + let + value = encoded[encPos] + offset = encoded[encPos + 1] + length = encoded[encPos + 2] + lengthIndex = (value shr 8) and (uint8.high shr 1) + distIndex = value and uint8.high + lengthExtraBits = baseLengthsExtraBits[lengthIndex] + lengthExtra = length - baseLengths[lengthIndex] + distExtraBits = baseDistanceExtraBits[distIndex] + distExtra = offset - baseDistance[distIndex] + inc(encPos, 3) + inc(srcPos, length.int) + + if b.data.len < b.bytePos + 6: + b.data.setLen(b.data.len * 2) + + b.addBits( + llCodes[lengthIndex + firstLengthCodeIndex], + llLengths[lengthIndex + firstLengthCodeIndex].int + ) + b.addBits(lengthExtra, lengthExtraBits) + b.addBits(distCodes[distIndex], distLengths[distIndex].int) + b.addBits(distExtra, distExtraBits) + else: + let length = encoded[encPos].int + inc encPos + + let worstCaseBytesNeeded = (length * maxLitLenCodeLength + 7) div 8 + if b.data.len < b.bytePos + worstCaseBytesNeeded: + b.data.setLen(max(b.bytePos + worstCaseBytesNeeded, b.data.len * 2)) + + for j in 0 ..< length: + b.addBits(llCodes[src[srcPos]], llLengths[src[srcPos]].int) + inc srcPos + + if llLengths[256] == 0: + failCompress() + + b.addBits(llCodes[256], llLengths[256].int) # End of block + + b.skipRemainingBitsInCurrentByte() b.data.setLen(b.bytePos) b.data