mirror of https://github.com/status-im/nimPNG.git
1334 lines
48 KiB
Nim
1334 lines
48 KiB
Nim
import streams, sequtils, algorithm, strutils
|
|
|
|
const
|
|
FIRST_LENGTH_CODE_INDEX = 257
|
|
LAST_LENGTH_CODE_INDEX = 285
|
|
#256 literals, the end code, some length codes, and 2 unused codes
|
|
NUM_DEFLATE_CODE_SYMBOLS = 288
|
|
#the distance codes have their own symbols, 30 used, 2 unused
|
|
NUM_DISTANCE_SYMBOLS = 32
|
|
#the code length codes.
|
|
#0-15: code lengths,
|
|
#16: copy previous 3-6 times,
|
|
#17: 3-10 zeros,
|
|
#18: 11-138 zeros
|
|
NUM_CODE_LENGTH_CODES = 19
|
|
|
|
#the base lengths represented by codes 257-285
|
|
LENGTHBASE = [3, 4, 5, 6, 7, 8, 9, 10,
|
|
11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51,
|
|
59, 67, 83, 99, 115, 131, 163, 195, 227, 258]
|
|
|
|
#the extra bits used by codes 257-285 (added to base length)
|
|
LENGTHEXTRA = [0, 0, 0, 0, 0, 0, 0, 0,
|
|
1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
|
|
4, 4, 4, 4, 5, 5, 5, 5, 0]
|
|
|
|
#the base backwards distances
|
|
#(the bits of distance codes appear after
|
|
#length codes and use their own huffman tree)
|
|
DISTANCEBASE = [1, 2, 3, 4, 5, 7, 9,
|
|
13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513,
|
|
769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577]
|
|
|
|
#the extra bits of backwards distances (added to base)
|
|
DISTANCEEXTRA = [0, 0, 0, 0, 1, 1, 2,
|
|
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
|
|
8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13]
|
|
|
|
#the order in which "code length alphabet code lengths" are stored,
|
|
#out of this the huffman tree of the dynamic huffman tree lengths is generated
|
|
CLCL_ORDER = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
|
|
|
|
#3 bytes of data get encoded into two bytes. The hash cannot use more than 3
|
|
#bytes as input because 3 is the minimum match length for deflate
|
|
HASH_NUM_VALUES = 65536
|
|
HASH_BIT_MASK = HASH_NUM_VALUES - 1
|
|
MAX_SUPPORTED_DEFLATE_LENGTH = 258
|
|
|
|
type
|
|
HuffmanTree = object
|
|
tree2d, tree1d: seq[int]
|
|
lengths: seq[int] #the lengths of the codes of the 1d-tree
|
|
maxbitlen: int #maximum number of bits a single code can get
|
|
numcodes: int #number of symbols in the alphabet = number of codes
|
|
|
|
BitStream = object
|
|
bitpointer: int
|
|
data: string
|
|
databitlen: int
|
|
|
|
NZError = ref object of Exception
|
|
|
|
NZHash = object
|
|
head: seq[int] #hash value to head circular pos
|
|
#can be outdated if went around window
|
|
chain: seq[int] #circular pos to prev circular pos
|
|
val: seq[int] #circular pos to hash value
|
|
|
|
#TODO: do this not only for zeros but for any repeated byte. However for PNG
|
|
#it's always going to be the zeros that dominate, so not important for PNG
|
|
|
|
headz: seq[int] #similar to head, but for chainz
|
|
chainz: seq[int] #those with same amount of zeros
|
|
zeros: seq[int] #length of zeros streak, used as a second hash chain
|
|
|
|
#A coin, this is the terminology used for the package-merge algorithm and the
|
|
#coin collector's problem. This is used to generate the huffman tree.
|
|
#A coin can be multiple coins (when they're merged)
|
|
|
|
Coin = ref object
|
|
symbols: seq[int]
|
|
weight: float #the sum of all weights in this coin
|
|
|
|
Coins = seq[Coin]
|
|
|
|
#Possible inflate modes between inflate() calls
|
|
#inflateMode = enum
|
|
# HEAD, # i: waiting for magic header
|
|
# FLAGS, # i: waiting for method and flags (gzip)
|
|
# TIME, # i: waiting for modification time (gzip)
|
|
# OS, # i: waiting for extra flags and operating system (gzip)
|
|
# EXLEN, # i: waiting for extra length (gzip)
|
|
# EXTRA, # i: waiting for extra bytes (gzip)
|
|
# NAME, # i: waiting for end of file name (gzip)
|
|
# COMMENT, # i: waiting for end of comment (gzip)
|
|
# HCRC, # i: waiting for header crc (gzip)
|
|
# DICTID, # i: waiting for dictionary check value
|
|
# DICT, # waiting for inflateSetDictionary() call
|
|
# TYPE, # i: waiting for type bits, including last-flag bit
|
|
# TYPEDO, # i: same, but skip check to exit inflate on new block
|
|
# STORED, # i: waiting for stored size (length and complement)
|
|
# COPY_FIRST, # i/o: same as COPY below, but only first time in
|
|
# COPY, # i/o: waiting for input or output to copy stored block
|
|
# TABLE, # i: waiting for dynamic block table lengths
|
|
# LENLENS, # i: waiting for code length code lengths
|
|
# CODELENS, # i: waiting for length/lit and distance code lengths
|
|
# LEN_FIRST, # i: same as LEN below, but only first time in
|
|
# LEN, # i: waiting for length/lit/eob code
|
|
# LENEXT, # i: waiting for length extra bits
|
|
# DIST, # i: waiting for distance code
|
|
# DISTEXT, # i: waiting for distance extra bits
|
|
# MATCH, # o: waiting for output space to copy string
|
|
# LIT, # o: waiting for output space to write literal
|
|
# CHECK, # i: waiting for 32-bit check value
|
|
# LENGTH, # i: waiting for 32-bit length (gzip)
|
|
# DONE, # finished check, done -- remain here until reset
|
|
# BAD, # got a data error -- remain here until reset
|
|
# MEM, # got an inflate() memory error -- remain here until reset
|
|
# SYNC # looking for synchronization bytes to restart inflate()
|
|
|
|
nzStreamMode = enum
|
|
nzsDeflate, nzsInflate
|
|
|
|
nzStream* = ref object
|
|
btype: range[0..3]
|
|
use_lz77: bool
|
|
windowsize: range[2..32768]
|
|
minmatch: range[3..258]
|
|
nicematch: range[3..358]
|
|
lazymatching: bool
|
|
bits: BitStream
|
|
data: string
|
|
mode: nzStreamMode
|
|
ignoreAdler32*: bool
|
|
|
|
proc newNZError(msg: string): NZError =
|
|
new(result)
|
|
result.msg = msg
|
|
|
|
proc readBit(s: BitStream): int {.inline.} =
|
|
result = (ord(s.data[s.bitpointer shr 3]) shr (s.bitpointer and 0x07)) and 0x01
|
|
|
|
proc readBitFromStream(s: var BitStream): int {.inline.} =
|
|
result = s.readBit
|
|
inc s.bitpointer
|
|
|
|
proc readBitsFromStream(s: var BitStream, nbits: int): int =
|
|
for i in 0..nbits-1:
|
|
inc(result, s.readBit shl i)
|
|
inc s.bitpointer
|
|
|
|
proc readBitsSafe(s: var BitStream, nbits: int): int =
|
|
if s.bitpointer + nbits > s.databitlen:
|
|
raise newNZError("bit pointer jumps past memory")
|
|
|
|
for i in 0..nbits-1:
|
|
inc(result, s.readBit shl i)
|
|
inc s.bitpointer
|
|
|
|
#the tree representation used by the decoder.
|
|
proc HuffmanTree_make2DTree(tree: var HuffmanTree) =
|
|
var nodefilled = 0 #up to which node it is filled
|
|
var treepos = 0 #position in the tree (1 of the numcodes columns)
|
|
|
|
#32767 here means the tree2d isn't filled there yet
|
|
tree.tree2d = newSeqWith(tree.numcodes * 2, 32767)
|
|
|
|
#convert tree1d[] to tree2d[][]. In the 2D array, a value of 32767 means
|
|
#uninited, a value >= numcodes is an address to another bit, a value < numcodes
|
|
#is a code. The 2 rows are the 2 possible bit values (0 or 1), there are as
|
|
#many columns as codes - 1.
|
|
#A good huffmann tree has N * 2 - 1 nodes, of which N - 1 are internal nodes.
|
|
#Here, the internal nodes are stored (what their 0 and 1 option point to).
|
|
#There is only memory for such good tree currently, if there are more nodes
|
|
#(due to too long length codes), error 55 will happen
|
|
|
|
for n in 0..tree.numcodes-1: #the codes
|
|
let len = tree.lengths[n]
|
|
for i in 0..len-1: #the bits for this code
|
|
let bit = (tree.tree1d[n] shr (len - i - 1)) and 1
|
|
let branch = 2 * treepos + bit
|
|
#oversubscribed, see comment in lodepng_error_text
|
|
if treepos > 2147483647 or treepos + 2 > tree.numcodes:
|
|
raise newNZError("oversubscribed")
|
|
|
|
if tree.tree2d[branch] != 32767: #not yet filled in
|
|
treepos = tree.tree2d[branch] - tree.numcodes
|
|
continue
|
|
|
|
if i + 1 < len:
|
|
#put address of the next step in here, first that address has to be found of course
|
|
#(it's just nodefilled + 1)...
|
|
inc(nodefilled)
|
|
#addresses encoded with numcodes added to it
|
|
tree.tree2d[branch] = nodefilled + tree.numcodes
|
|
treepos = nodefilled
|
|
continue
|
|
|
|
#last bit
|
|
tree.tree2d[branch] = n #put the current code in it
|
|
treepos = 0 #start from root again
|
|
|
|
tree.tree2d.applyIt(if it == 32767: 0 else: it) #remove possible remaining 32767's
|
|
|
|
#Second step for the ...makeFromLengths and ...makeFromFrequencies functions.
|
|
#numcodes, lengths and maxbitlen must already be filled in correctly.
|
|
proc HuffmanTree_makeFromLengths2(tree: var HuffmanTree) =
|
|
tree.tree1d = newSeq[int](tree.numcodes)
|
|
var blcount = newSeq[int](tree.maxbitlen + 1)
|
|
var nextcode = newSeq[int](tree.maxbitlen + 1)
|
|
|
|
#step 1: count number of instances of each code length
|
|
for len in tree.lengths: inc blcount[len]
|
|
|
|
#step 2: generate the nextcode values
|
|
for bits in 1..tree.maxbitlen:
|
|
nextcode[bits] = (nextcode[bits - 1] + blcount[bits - 1]) shl 1
|
|
|
|
#step 3: generate all the codes
|
|
for n in 0..tree.numcodes-1:
|
|
let len = tree.lengths[n]
|
|
if len != 0:
|
|
tree.tree1d[n] = nextcode[len]
|
|
inc nextcode[len]
|
|
|
|
#given the code lengths (as stored in the compressed data),
|
|
#generate the tree as defined by Deflate.
|
|
#maxbitlen is the maximum bits that a code in the tree can have.
|
|
proc HuffmanTree_makeFromLengths(tree: var HuffmanTree, bitlen: openarray[int], maxbitlen: int) =
|
|
tree.lengths = @bitlen
|
|
tree.numcodes = bitlen.len #number of symbols
|
|
tree.maxbitlen = maxbitlen
|
|
HuffmanTree_makeFromLengths2(tree)
|
|
HuffmanTree_make2DTree(tree)
|
|
|
|
proc make_coin(): Coin =
|
|
new(result)
|
|
result.symbols = @[]
|
|
|
|
proc coin_copy(c1, c2: Coin) =
|
|
c1.weight = c2.weight
|
|
c1.symbols = c2.symbols
|
|
|
|
proc add_coins(c1, c2: Coin) =
|
|
for sym in c2.symbols: c1.symbols.add sym
|
|
c1.weight += c2.weight
|
|
|
|
proc init_coins(c: var Coins, num: int) =
|
|
for i in 0..num-1: c[i] = make_coin()
|
|
|
|
proc cleanup_coins(c: var Coins, num: int) =
|
|
for i in 0..num-1: c[i].symbols = @[]
|
|
|
|
proc cmpx(a, b: Coin): int {.locks:0.} =
|
|
var wa = a.weight
|
|
var wb = b.weight
|
|
if wa > wb: result = 1
|
|
elif wa < wb: result = -1
|
|
else: result = 0
|
|
|
|
proc append_symbol_coins(coins: Coins, start: int, frequencies: openarray[int], numcodes, sum: int) =
|
|
var j = start #index of present symbols
|
|
for i in 0..numcodes-1:
|
|
if frequencies[i] != 0: #only include symbols that are present
|
|
coins[j].weight = frequencies[i] / sum
|
|
coins[j].symbols.add i
|
|
inc j
|
|
|
|
proc placePivot[T](a: var openArray[T], lo, hi: int): int {.locks:0.} =
|
|
var pivot = lo #set pivot
|
|
var switch_i = lo + 1
|
|
let x = lo+1
|
|
|
|
for i in x..hi: #run on array
|
|
if cmpx(a[i], a[pivot]) <= 0: #compare pivot and i
|
|
swap(a[i], a[switch_i]) #swap i and i to switch
|
|
swap(a[pivot], a[switch_i]) #swap pivot and i to switch
|
|
inc pivot #set current location of pivot
|
|
inc switch_i #set location for i to switch with pivot
|
|
result = pivot #return pivot location
|
|
|
|
proc quickSort[T](a: var openArray[T], lo, hi: int) {.locks:0.} =
|
|
if lo >= hi: return #stop condition
|
|
#set pivot location
|
|
var pivot = placePivot(a, lo, hi)
|
|
quickSort(a, lo, pivot-1) #sort bottom half
|
|
quickSort(a, pivot+1, hi) #sort top half
|
|
|
|
proc quickSort[T](a: var openArray[T], length = -1) {.locks:0.} =
|
|
var lo = 0
|
|
var hi = if length < 0: a.high else: length-1
|
|
quickSort(a, lo, hi)
|
|
|
|
proc huffman_code_lengths(frequencies: openarray[int], numcodes, maxbitlen: int): seq[int] =
|
|
var
|
|
lengths = newSeq[int](numcodes)
|
|
sum = 0
|
|
numpresent = 0
|
|
coins: Coins #the coins of the currently calculated row
|
|
prev_row: Coins #the previous row of coins
|
|
coinmem, numcoins: int
|
|
|
|
if numcodes == 0:
|
|
raise newNZError("a tree of 0 symbols is not supposed to be made")
|
|
|
|
for i in 0..numcodes-1:
|
|
if frequencies[i] > 0:
|
|
inc numpresent
|
|
inc(sum, frequencies[i])
|
|
|
|
#ensure at least two present symbols. There should be at least one symbol
|
|
#according to RFC 1951 section 3.2.7. To decoders incorrectly require two. To
|
|
#make these work as well ensure there are at least two symbols. The
|
|
#Package-Merge code below also doesn't work correctly if there's only one
|
|
#symbol, it'd give it the theoritical 0 bits but in practice zlib wants 1 bit
|
|
|
|
if numpresent == 0:
|
|
lengths[0] = 1
|
|
lengths[1] = 1 #note that for RFC 1951 section 3.2.7, only lengths[0] = 1 is needed
|
|
elif numpresent == 1:
|
|
for i in 0..numcodes-1:
|
|
if frequencies[i] != 0:
|
|
lengths[i] = 1
|
|
lengths[if i == 0: 1 else: 0] = 1
|
|
break
|
|
else:
|
|
#Package-Merge algorithm represented by coin collector's problem
|
|
#For every symbol, maxbitlen coins will be created
|
|
coinmem = numpresent * 2 #max amount of coins needed with the current algo
|
|
coins = newSeq[Coin](coinmem)
|
|
prev_row = newSeq[Coin](coinmem)
|
|
|
|
coins.init_coins(coinmem)
|
|
prev_row.init_coins(coinmem)
|
|
|
|
#first row, lowest denominator
|
|
append_symbol_coins(coins, 0, frequencies, numcodes, sum)
|
|
numcoins = numpresent
|
|
|
|
coins.quickSort(numcoins)
|
|
|
|
var numprev = 0
|
|
for j in 1..maxbitlen: #each of the remaining rows
|
|
swap(prev_row, coins)
|
|
swap(numprev, numcoins)
|
|
|
|
coins.cleanup_coins(numcoins)
|
|
coins.init_coins(numcoins)
|
|
numcoins = 0
|
|
|
|
#fill in the merged coins of the previous row
|
|
var i = 0
|
|
while i + 1 < numprev:
|
|
#merge prev_row[i] and prev_row[i + 1] into new coin
|
|
var coin = coins[numcoins]
|
|
coin_copy(coin, prev_row[i])
|
|
add_coins(coin, prev_row[i + 1])
|
|
inc numcoins
|
|
inc(i, 2)
|
|
|
|
#fill in all the original symbols again
|
|
if j < maxbitlen:
|
|
append_symbol_coins(coins, numcoins, frequencies, numcodes, sum)
|
|
inc(numcoins, numpresent)
|
|
|
|
coins.quickSort(numcoins)
|
|
|
|
#calculate the lengths of each symbol, as the amount of times a coin of each symbol is used
|
|
var i = 0
|
|
while i + 1 < numpresent:
|
|
var coin = coins[i]
|
|
for j in 0..coin.symbols.high: inc lengths[coin.symbols[j]]
|
|
inc i
|
|
result = lengths
|
|
|
|
#Create the Huffman tree given the symbol frequencies
|
|
proc HuffmanTree_makeFromFrequencies(
|
|
tree: var HuffmanTree, frequencies: openarray[int], mincodes, maxbitlen: int) =
|
|
var numcodes = frequencies.len
|
|
while(frequencies[numcodes - 1] == 0) and (numcodes > mincodes):
|
|
dec numcodes #trim zeroes
|
|
|
|
tree.maxbitlen = maxbitlen
|
|
tree.numcodes = numcodes #number of symbols
|
|
tree.lengths = huffman_code_lengths(frequencies, numcodes, maxbitlen)
|
|
HuffmanTree_makeFromLengths2(tree)
|
|
|
|
#get the literal and length code tree of a deflated block with fixed tree,
|
|
#as per the deflate specification
|
|
proc generateFixedLitLenTree(tree: var HuffmanTree) =
|
|
var bitlen: array[0..NUM_DEFLATE_CODE_SYMBOLS-1, int]
|
|
|
|
#288 possible codes:
|
|
#0-255=literals, 256=endcode, 257-285=lengthcodes, 286-287=unused
|
|
for i in 0..143: bitlen[i] = 8
|
|
for i in 144..255: bitlen[i] = 9
|
|
for i in 256..279: bitlen[i] = 7
|
|
for i in 280..287: bitlen[i] = 8
|
|
|
|
HuffmanTree_makeFromLengths(tree, bitlen, 15)
|
|
|
|
proc generateFixedDistanceTree(tree: var HuffmanTree) =
|
|
var bitlen: array[0..NUM_DISTANCE_SYMBOLS-1, int]
|
|
|
|
#there are 32 distance codes, but 30-31 are unused
|
|
for i in 0..bitlen.len-1: bitlen[i] = 5
|
|
HuffmanTree_makeFromLengths(tree, bitlen, 15)
|
|
|
|
proc readInt16(s: var BitStream): int =
|
|
#go to first boundary of byte
|
|
while (s.bitpointer and 0x7) != 0: inc s.bitpointer
|
|
var p = s.bitpointer div 8 #byte position
|
|
if p + 2 >= s.data.len: raise newNZError("bit pointer will jump past memory")
|
|
result = ord(s.data[p]) + 256 * ord(s.data[p + 1])
|
|
inc(s.bitpointer, 16)
|
|
|
|
proc getBytePosition(s: var BitStream): int =
|
|
result = s.bitpointer div 8 #byte position
|
|
|
|
proc readByte(s: var BitStream): int =
|
|
while (s.bitpointer and 0x7) != 0: inc s.bitpointer
|
|
var p = s.bitpointer div 8 #byte position
|
|
if p + 1 >= s.data.len: raise newNZError("bit pointer will jump past memory")
|
|
result = ord(s.data[p])
|
|
inc(s.bitpointer, 8)
|
|
|
|
proc inflateNoCompression(nz: nzStream) =
|
|
let inlength = nz.bits.data.len
|
|
|
|
#read LEN (2 bytes) and NLEN (2 bytes)
|
|
let LEN = nz.bits.readInt16
|
|
let NLEN = nz.bits.readInt16
|
|
|
|
#check if 16-bit NLEN is really the one's complement of LEN
|
|
if LEN + NLEN != 65535:
|
|
raise newNZError("NLEN is not one's complement of LEN")
|
|
|
|
#read the literal data: LEN bytes are now stored in the out buffer
|
|
var p = nz.bits.getBytePosition
|
|
if p + LEN > inlength:
|
|
raise newNZError("reading outside of input buffer")
|
|
|
|
var pos = nz.data.len
|
|
nz.data.setLen(pos + LEN)
|
|
for i in 0..LEN-1:
|
|
nz.data[pos] = nz.bits.data[p]
|
|
inc pos
|
|
inc p
|
|
|
|
nz.bits.bitpointer = p * 8
|
|
|
|
#get the tree of a deflated block with fixed tree,
|
|
#as specified in the deflate specification
|
|
proc getTreeInflateFixed(tree_ll, tree_d: var HuffmanTree) =
|
|
generateFixedLitLenTree(tree_ll)
|
|
generateFixedDistanceTree(tree_d)
|
|
|
|
#returns the code, or (unsigned)(-1) if error happened
|
|
#inbitlength is the length of the complete buffer, in bits (so its byte length times 8)
|
|
|
|
proc huffmanDecodeSymbol(s: var BitStream, codetree: HuffmanTree, inbitlength: int): int =
|
|
var treepos = 0
|
|
|
|
while true:
|
|
if s.bitpointer >= inbitlength:
|
|
return -1 #end of input memory reached without endcode
|
|
|
|
#decode the symbol from the tree. The "readBitFromStream" code is inlined in
|
|
#the expression below because this is the biggest bottleneck while decoding
|
|
let ct = codetree.tree2d[(treepos shl 1) + s.readBit]
|
|
inc s.bitpointer
|
|
if ct < codetree.numcodes: return ct #the symbol is decoded, return it
|
|
else: treepos = ct - codetree.numcodes #symbol not yet decoded, instead move tree position
|
|
|
|
if treepos >= codetree.numcodes: return -1 #it appeared outside the codetree
|
|
|
|
proc getTreeInflateDynamic(s: var BitStream, tree_ll, tree_d: var HuffmanTree) =
|
|
#make sure that length values that aren't filled in will be 0,
|
|
#or a wrong tree will be generated
|
|
let inlength = s.data.len
|
|
let inbitlength = inlength * 8
|
|
|
|
#see comments in deflateDynamic for explanation
|
|
#of the context and these variables, it is analogous
|
|
var bitlen_ll = newSeq[int](NUM_DEFLATE_CODE_SYMBOLS) #lit,len code lengths
|
|
var bitlen_d = newSeq[int](NUM_DISTANCE_SYMBOLS) #dist code lengths
|
|
|
|
#code length code lengths ("clcl"),
|
|
#the bit lengths of the huffman tree
|
|
#used to compress bitlen_ll and bitlen_d
|
|
var bitlen_cl = newSeq[int](NUM_CODE_LENGTH_CODES)
|
|
|
|
#the code tree for code length codes
|
|
#(the huffman tree for compressed huffman trees)
|
|
var tree_cl: HuffmanTree
|
|
|
|
if s.bitpointer + 14 > inbitlength:
|
|
raise newNZError("the bit pointer is or will go past the memory")
|
|
|
|
#number of literal/length codes + 257.
|
|
#Unlike the spec, the value 257 is added to it here already
|
|
let HLIT = s.readBitsFromStream(5) + 257
|
|
#number of distance codes.
|
|
#Unlike the spec, the value 1 is added to it here already
|
|
let HDIST = s.readBitsFromStream(5) + 1
|
|
|
|
#number of code length codes.
|
|
#Unlike the spec, the value 4 is added to it here already
|
|
let HCLEN = s.readBitsFromStream(4) + 4
|
|
|
|
if s.bitpointer + HCLEN * 3 > inbitlength:
|
|
raise newNZError("the bit pointer is or will go past the memory")
|
|
|
|
#read the code length codes out of 3 * (amount of code length codes) bits
|
|
for i in 0..NUM_CODE_LENGTH_CODES-1:
|
|
if i < HCLEN: bitlen_cl[CLCL_ORDER[i]] = s.readBitsFromStream(3)
|
|
else: bitlen_cl[CLCL_ORDER[i]] = 0 #if not, it must stay 0
|
|
|
|
HuffmanTree_makeFromLengths(tree_cl, bitlen_cl, 7)
|
|
#now we can use this tree to read the lengths
|
|
#for the tree that this function will return
|
|
|
|
#i is the current symbol we're reading in the part
|
|
#that contains the code lengths of lit/len and dist codes
|
|
var i = 0
|
|
while i < HLIT + HDIST:
|
|
let code = s.huffmanDecodeSymbol(tree_cl, inbitlength)
|
|
if code <= 15: #a length code
|
|
if i < HLIT: bitlen_ll[i] = code
|
|
else: bitlen_d[i - HLIT] = code
|
|
inc(i)
|
|
elif code == 16: #repeat previous
|
|
var replength = 3 #read in the 2 bits that indicate repeat length (3-6)
|
|
var value = 0 #set value to the previous code
|
|
|
|
if i == 0: raise newNZError("can't repeat previous if i is 0")
|
|
replength += s.readBitsSafe(2)
|
|
|
|
if i < HLIT + 1: value = bitlen_ll[i - 1]
|
|
else: value = bitlen_d[i - HLIT - 1]
|
|
|
|
#repeat this value in the next lengths
|
|
for n in 0..replength-1:
|
|
if i >= HLIT + HDIST: raise newNZError("i is larger than the amount of codes")
|
|
if i < HLIT: bitlen_ll[i] = value
|
|
else: bitlen_d[i - HLIT] = value
|
|
inc(i)
|
|
elif code == 17: #repeat "0" 3-10 times
|
|
var replength = 3 #read in the bits that indicate repeat length
|
|
replength += s.readBitsSafe(3)
|
|
|
|
#repeat this value in the next lengths
|
|
for n in 0..replength-1:
|
|
if i >= HLIT + HDIST: raise newNZError("i is larger than the amount of codes")
|
|
if i < HLIT: bitlen_ll[i] = 0
|
|
else: bitlen_d[i - HLIT] = 0
|
|
inc(i)
|
|
elif code == 18: #repeat "0" 11-138 times
|
|
var replength = 11 #read in the bits that indicate repeat length
|
|
replength += s.readBitsSafe(7)
|
|
|
|
#repeat this value in the next lengths
|
|
for n in 0..replength-1:
|
|
if i >= HLIT + HDIST: raise newNZError("i is larger than the amount of codes")
|
|
if i < HLIT: bitlen_ll[i] = 0
|
|
else: bitlen_d[i - HLIT] = 0
|
|
inc(i)
|
|
else: #if(code == -1) huffmanDecodeSymbol returns -1 in case of error
|
|
if code == -1:
|
|
#return error code 10 or 11 depending on the situation that happened in huffmanDecodeSymbol
|
|
#(10=no endcode, 11=wrong jump outside of tree)
|
|
if s.bitpointer > inbitlength: raise newNZError("no endcode")
|
|
else: raise newNZError("wrong jump outside of tree")
|
|
else:
|
|
raise newNZError("unexisting code, this can never happen")
|
|
break
|
|
|
|
if bitlen_ll[256] == 0:
|
|
raise newNZError("the length of the end code 256 must be larger than 0")
|
|
|
|
#now we've finally got HLIT and HDIST,
|
|
#so generate the code trees, and the function is done
|
|
HuffmanTree_makeFromLengths(tree_ll, bitlen_ll, 15)
|
|
HuffmanTree_makeFromLengths(tree_d, bitlen_d, 15)
|
|
|
|
#inflate a block with dynamic or fixed Huffman tree
|
|
proc inflateHuffmanBlock(nz: nzStream, blockType: int) =
|
|
var tree_ll: HuffmanTree #the huffman tree for literal and length codes
|
|
var tree_d: HuffmanTree #the huffman tree for distance codes
|
|
let inlength = nz.bits.data.len
|
|
let inbitlength = inlength * 8
|
|
|
|
if blockType == 1: getTreeInflateFixed(tree_ll, tree_d)
|
|
elif blockType == 2: nz.bits.getTreeInflateDynamic(tree_ll, tree_d)
|
|
|
|
#decode all symbols until end reached, breaks at end code
|
|
#code_ll is literal, length or end code
|
|
while true:
|
|
let code_ll = nz.bits.huffmanDecodeSymbol(tree_ll, inbitlength)
|
|
if code_ll <= 255: #literal symbol
|
|
nz.data.add chr(code_ll)
|
|
elif code_ll >= FIRST_LENGTH_CODE_INDEX and code_ll <= LAST_LENGTH_CODE_INDEX: #length code
|
|
#part 1: get length base
|
|
var length = LENGTHBASE[code_ll - FIRST_LENGTH_CODE_INDEX]
|
|
|
|
#part 2: get extra bits and add the value of that to length
|
|
let numextrabits_l = LENGTHEXTRA[code_ll - FIRST_LENGTH_CODE_INDEX]
|
|
length += nz.bits.readBitsSafe(numextrabits_l)
|
|
|
|
#part 3: get distance code
|
|
let code_d = nz.bits.huffmanDecodeSymbol(tree_d, inbitlength)
|
|
if code_d > 29:
|
|
if code_ll == -1: #huffmanDecodeSymbol returns -1 in case of error
|
|
#return error code 10 or 11 depending on the situation that happened in huffmanDecodeSymbol
|
|
#(10=no endcode, 11=wrong jump outside of tree)
|
|
if nz.bits.bitpointer > inbitlength: raise newNZError("no endcode")
|
|
else: raise newNZError("wrong jump outside of tree")
|
|
else:
|
|
raise newNZError("invalid distance code (30-31 are never used)")
|
|
break
|
|
var distance = DISTANCEBASE[code_d]
|
|
|
|
#part 4: get extra bits from distance
|
|
let numextrabits_d = DISTANCEEXTRA[code_d]
|
|
distance += nz.bits.readBitsSafe(numextrabits_d)
|
|
|
|
#part 5: fill in all the out[n] values based on the length and dist
|
|
let start = nz.data.len
|
|
if distance > start:
|
|
raise newNZError("too long backward distance")
|
|
var backward = start - distance
|
|
|
|
nz.data.setLen(start + length)
|
|
for pos in 0..length-1:
|
|
nz.data[pos+start] = nz.data[backward]
|
|
inc backward
|
|
if backward >= start: backward = start - distance
|
|
elif code_ll == 256:
|
|
break #end code, break the loop
|
|
else: #if(code == -1) huffmanDecodeSymbol returns -1 in case of error
|
|
#return error code 10 or 11 depending on the situation that happened in huffmanDecodeSymbol
|
|
#(10=no endcode, 11=wrong jump outside of tree)
|
|
if nz.bits.bitpointer > inbitlength: raise newNZError("no endcode")
|
|
else: raise newNZError("wrong jump outside of tree")
|
|
break
|
|
|
|
proc nzInflate(nz: nzStream) =
|
|
var finalBlock = false
|
|
var streamLen = nz.bits.databitlen
|
|
|
|
while not finalBlock:
|
|
if nz.bits.bitpointer + 2 >= streamLen: break
|
|
#error, bit pointer will jump past memory
|
|
|
|
finalBlock = nz.bits.readBitFromStream != 0
|
|
let blockType = nz.bits.readBitFromStream + 2 * nz.bits.readBitFromStream
|
|
|
|
if blockType == 3: raise newNZError("invalid blockType")
|
|
elif blockType == 0: nz.inflateNoCompression #no compression
|
|
else: nz.inflateHuffmanBlock(blockType) #compression, blockType 01 or 10
|
|
|
|
proc nimzHashInit(hash: var NZHash, windowsize: int) =
|
|
hash.head = newSeqWith(HASH_NUM_VALUES, -1)
|
|
hash.val = newSeqWith(windowsize, -1)
|
|
hash.chain = newSeq[int](windowsize)
|
|
hash.zeros = newSeq[int](windowsize)
|
|
hash.headz = newSeqWith(MAX_SUPPORTED_DEFLATE_LENGTH + 1, -1)
|
|
hash.chainz = newSeq[int](windowsize)
|
|
for i in 0..windowsize-1:
|
|
hash.chain[i] = i
|
|
hash.chainz[i] = i
|
|
|
|
proc deflateNoCompression(nz: nzStream) =
|
|
#non compressed deflate block data:
|
|
#1 bit BFINAL,2 bits BTYPE,(5 bits): it jumps to start of next byte,
|
|
#2 bytes LEN, 2 bytes NLEN, LEN bytes literal DATA
|
|
|
|
let datasize = nz.data.len
|
|
let numdeflateblocks = (datasize + 65534) div 65535
|
|
var datapos = 0
|
|
|
|
for i in 0..numdeflateblocks-1:
|
|
let finalBlock = (i == numdeflateblocks - 1)
|
|
nz.bits.data.add chr(if finalBlock: 1 else: 0)
|
|
|
|
var LEN = 65535
|
|
if datasize - datapos < 65535: LEN = datasize - datapos
|
|
let NLEN = 65535 - LEN
|
|
|
|
nz.bits.data.add chr(LEN mod 256)
|
|
nz.bits.data.add chr(LEN div 256)
|
|
nz.bits.data.add chr(NLEN mod 256)
|
|
nz.bits.data.add chr(NLEN div 256)
|
|
|
|
#Decompressed data
|
|
var j = 0
|
|
while j < 65535 and datapos < datasize:
|
|
nz.bits.data.add nz.data[datapos]
|
|
inc datapos
|
|
inc j
|
|
|
|
proc `|=`(a: var char, b: char) {.inline.} =
|
|
a = chr(ord(a) or ord(b))
|
|
|
|
proc addBitToStream(s: var BitStream, bit: int) =
|
|
#add a new byte at the end
|
|
if (s.bitpointer and 0x07) == 0: s.data.add chr(0)
|
|
#earlier bit of huffman code is in a lesser significant bit of an earlier byte
|
|
s.data[s.data.len - 1] |= chr(bit shl (s.bitpointer and 0x07))
|
|
inc s.bitpointer
|
|
|
|
proc addBitsToStream(s: var BitStream, value: int, nbits: int) =
|
|
for i in 0..nbits-1:
|
|
s.addBitToStream((value shr i) and 1)
|
|
|
|
proc addBitsToStreamReversed(s: var BitStream, value: int, nbits: int) =
|
|
for i in 0..nbits-1:
|
|
s.addBitToStream((value shr (nbits - 1 - i)) and 1)
|
|
|
|
proc HuffmanTree_getCode(tree: HuffmanTree, index: int): int =
|
|
result = tree.tree1d[index]
|
|
|
|
proc HuffmanTree_getLength(tree: HuffmanTree, index: int): int =
|
|
result = tree.lengths[index]
|
|
|
|
proc addHuffmanSymbol(s: var BitStream, tree: HuffmanTree, val: int) {.inline.} =
|
|
s.addBitsToStreamReversed(
|
|
HuffmanTree_getCode(tree, val),
|
|
HuffmanTree_getLength(tree, val))
|
|
|
|
#write the lz77-encoded data, which has lit, len and dist codes, to compressed stream using huffman trees.
|
|
#tree_ll: the tree for lit and len codes.
|
|
#tree_d: the tree for distance codes.
|
|
proc writeLZ77data(s: var BitStream, input: seq[int], tree_ll, tree_d: HuffmanTree) =
|
|
var i = 0
|
|
while i < input.len:
|
|
let val = input[i]
|
|
s.addHuffmanSymbol(tree_ll, val)
|
|
if val > 256: #for a length code, 3 more things have to be added
|
|
let length_index = val - FIRST_LENGTH_CODE_INDEX
|
|
let n_length_extra_bits = LENGTHEXTRA[length_index]
|
|
let length_extra_bits = input[i+1]
|
|
let distance_code = input[i+2]
|
|
let n_distance_extra_bits = DISTANCEEXTRA[distance_code]
|
|
let distance_extra_bits = input[i+3]
|
|
inc(i, 3)
|
|
|
|
s.addBitsToStream(length_extra_bits, n_length_extra_bits)
|
|
s.addHuffmanSymbol(tree_d, distance_code)
|
|
s.addBitsToStream(distance_extra_bits, n_distance_extra_bits)
|
|
inc i
|
|
|
|
proc `^=`(a: var int, b: int) =
|
|
a = a xor b
|
|
|
|
proc getHash(nz: nzStream, size, pos: int): int =
|
|
if pos + 2 < size:
|
|
#simple shift and xor hash is used. Since the data of PNGs is dominated
|
|
#by zeroes due to the filters, a better hash does not have a significant
|
|
#effect on speed in traversing the chain, and causes more time spend on
|
|
#calculating the hash.
|
|
result ^= (ord(nz.data[pos + 0]) shl 0)
|
|
result ^= (ord(nz.data[pos + 1]) shl 4)
|
|
result ^= (ord(nz.data[pos + 2]) shl 8)
|
|
else:
|
|
if pos >= size: return 0
|
|
let amount = size - pos
|
|
for i in 0..amount-1: result ^= (ord(nz.data[pos + i]) shl (i * 8))
|
|
|
|
result = result and HASH_BIT_MASK
|
|
|
|
proc countZeros(nz: nzStream, size, pos: int): int =
|
|
var datapos = pos
|
|
var dataend = min(datapos + MAX_SUPPORTED_DEFLATE_LENGTH, size)
|
|
while datapos < dataend and nz.data[datapos] == chr(0): inc datapos
|
|
#subtracting two addresses returned as 32-bit number (max value is MAX_SUPPORTED_DEFLATE_LENGTH)
|
|
result = datapos - pos
|
|
|
|
#wpos = pos & (windowsize - 1)
|
|
proc updateHashChain(hash: var NZHash, wpos, hashval, numzeros: int) =
|
|
hash.val[wpos] = hashval
|
|
if hash.head[hashval] != -1: hash.chain[wpos] = hash.head[hashval]
|
|
hash.head[hashval] = wpos
|
|
|
|
hash.zeros[wpos] = numzeros
|
|
if hash.headz[numzeros] != -1: hash.chainz[wpos] = hash.headz[numzeros]
|
|
hash.headz[numzeros] = wpos
|
|
|
|
proc getMaxChainLen(nz: nzStream): int =
|
|
result = if nz.windowsize >= 8192: nz.windowsize else: nz.windowsize div 8
|
|
|
|
proc getMaxLazyMatch(nz:nzStream): int =
|
|
result = if nz.windowsize >= 8192: MAX_SUPPORTED_DEFLATE_LENGTH else: 64
|
|
|
|
#search the index in the array, that has the largest value smaller than or equal to the given value,
|
|
#given array must be sorted (if no value is smaller, it returns the size of the given array)
|
|
proc searchCodeIndex(input: openarray[int], value: int): int =
|
|
#linear search implementation
|
|
#for i in 1..high(input):
|
|
#if input[i] > value: return i - 1
|
|
#return input.len - 1
|
|
|
|
#binary search implementation (not that much faster) (precondition: array_size > 0)
|
|
var left = 1
|
|
var right = input.len - 1
|
|
while left <= right:
|
|
let mid = (left + right) div 2
|
|
if input[mid] <= value: left = mid + 1 #the value to find is more to the right
|
|
elif input[mid - 1] > value: right = mid - 1 #the value to find is more to the left
|
|
else: return mid - 1
|
|
result = input.len - 1
|
|
|
|
proc addLengthDistance(values: var seq[int], length, distance: int) =
|
|
#values in encoded vector are those used by deflate:
|
|
#0-255: literal bytes
|
|
#256: end
|
|
#257-285: length/distance pair
|
|
#(length code, followed by extra length bits, distance code, extra distance bits)
|
|
#286-287: invalid
|
|
|
|
let length_code = searchCodeIndex(LENGTHBASE, length)
|
|
let extra_length = length - LENGTHBASE[length_code]
|
|
let dist_code = searchCodeIndex(DISTANCEBASE, distance)
|
|
let extra_distance = distance - DISTANCEBASE[dist_code]
|
|
|
|
values.add(length_code + FIRST_LENGTH_CODE_INDEX)
|
|
values.add extra_length
|
|
values.add dist_code
|
|
values.add extra_distance
|
|
|
|
#LZ77-encode the data. Return value is error code. The input are raw bytes, the output
|
|
#is in the form of unsigned integers with codes representing for example literal bytes, or
|
|
#length/distance pairs.
|
|
#It uses a hash table technique to let it encode faster. When doing LZ77 encoding, a
|
|
#sliding window (of windowsize) is used, and all past bytes in that window can be used as
|
|
#the "dictionary". A brute force search through all possible distances would be slow, and
|
|
#this hash technique is one out of several ways to speed this up.
|
|
proc encodeLZ77(nz: nzStream, hash: var NZHash, inpos, insize: int): seq[int] =
|
|
#for large window lengths, assume the user wants no compression loss.
|
|
#Otherwise, max hash chain length speedup.
|
|
result = @[]
|
|
|
|
var maxchainlength = nz.getMaxChainLen
|
|
var maxlazymatch = nz.getMaxLazyMatch
|
|
|
|
#not sure if setting it to false for windowsize < 8192 is better or worse
|
|
var
|
|
usezeros = true
|
|
numzeros = 0
|
|
lazy = 0
|
|
lazylength = 0
|
|
lazyoffset = 0
|
|
hashval: int
|
|
offset, length: int
|
|
hashpos: int
|
|
lastptr, foreptr, backptr: int
|
|
prev_offset: int
|
|
current_offset, current_length: int
|
|
|
|
if (nz.windowsize == 0) or (nz.windowsize > 32768):
|
|
raise newNZError("windowsize smaller/larger than allowed")
|
|
if (nz.windowsize and (nz.windowsize - 1)) != 0:
|
|
raise newNZError("must be power of two")
|
|
|
|
var nicematch = min(nz.nicematch, MAX_SUPPORTED_DEFLATE_LENGTH)
|
|
var pos = inpos
|
|
|
|
while pos < insize:
|
|
var wpos = pos and (nz.windowsize - 1) #position for in 'circular' hash buffers
|
|
var chainlength = 0
|
|
hashval = getHash(nz, insize, pos)
|
|
|
|
if usezeros and hashval == 0:
|
|
if numzeros == 0: numzeros = countZeros(nz, insize, pos)
|
|
elif (pos + numzeros > insize) or (nz.data[pos + numzeros - 1] != chr(0)): dec numzeros
|
|
else: numzeros = 0
|
|
|
|
updateHashChain(hash, wpos, hashval, numzeros)
|
|
|
|
#the length and offset found for the current position
|
|
length = 0
|
|
offset = 0
|
|
hashpos = hash.chain[wpos]
|
|
lastptr = min(insize, pos + MAX_SUPPORTED_DEFLATE_LENGTH)
|
|
|
|
#search for the longest string
|
|
prev_offset = 0
|
|
while true:
|
|
if chainlength >= maxchainlength: break
|
|
inc chainlength
|
|
current_offset = if hashpos <= wpos: wpos - hashpos else: wpos - hashpos + nz.windowsize
|
|
|
|
#stop when went completely around the circular buffer
|
|
if current_offset < prev_offset: break
|
|
prev_offset = current_offset
|
|
if current_offset > 0:
|
|
#test the next characters
|
|
foreptr = pos
|
|
backptr = pos - current_offset
|
|
|
|
#common case in PNGs is lots of zeros. Quickly skip over them as a speedup
|
|
if numzeros >= 3:
|
|
let skip = min(numzeros, hash.zeros[hashpos])
|
|
inc(backptr, skip)
|
|
inc(foreptr, skip)
|
|
|
|
#maximum supported length by deflate is max length
|
|
while foreptr < lastptr:
|
|
if nz.data[backptr] != nz.data[foreptr]: break
|
|
inc backptr
|
|
inc foreptr
|
|
|
|
current_length = foreptr - pos
|
|
|
|
if current_length > length:
|
|
length = current_length #the longest length
|
|
offset = current_offset #the offset that is related to this longest length
|
|
#jump out once a length of max length is found (speed gain). This also jumps
|
|
#out if length is MAX_SUPPORTED_DEFLATE_LENGTH
|
|
if current_length >= nicematch: break
|
|
|
|
if hashpos == hash.chain[hashpos]: break
|
|
|
|
if (numzeros >= 3) and (length > numzeros):
|
|
hashpos = hash.chainz[hashpos]
|
|
if hash.zeros[hashpos] != numzeros: break
|
|
else:
|
|
hashpos = hash.chain[hashpos]
|
|
#outdated hash value, happens if particular
|
|
#value was not encountered in whole last window
|
|
if hash.val[hashpos] != hashval: break
|
|
|
|
if nz.lazymatching:
|
|
if (lazy==0) and (length >= 3) and (length <= maxlazymatch) and (length < MAX_SUPPORTED_DEFLATE_LENGTH):
|
|
lazy = 1
|
|
lazylength = length
|
|
lazyoffset = offset
|
|
inc pos
|
|
continue #try the next byte
|
|
|
|
if lazy != 0:
|
|
lazy = 0
|
|
if pos == 0: raise newNZError("lazy matching at pos 0 is impossible")
|
|
if length > lazylength + 1:
|
|
#push the previous character as literal
|
|
result.add ord(nz.data[pos - 1])
|
|
else:
|
|
length = lazylength
|
|
offset = lazyoffset
|
|
hash.head[hashval] = -1 #the same hashchain update will be done, this ensures no wrong alteration*
|
|
hash.headz[numzeros] = -1 #idem
|
|
dec pos
|
|
|
|
if(length >= 3) and (offset > nz.windowsize):
|
|
raise newNZError("too big (or overflown negative) offset")
|
|
|
|
#encode it as length/distance pair or literal value
|
|
if length < 3: #only lengths of 3 or higher are supported as length/distance pair
|
|
result.add ord(nz.data[pos])
|
|
elif(length < nz.minmatch) or ((length == 3) and (offset > 4096)):
|
|
#compensate for the fact that longer offsets have more extra bits, a
|
|
#length of only 3 may be not worth it then
|
|
result.add ord(nz.data[pos])
|
|
else:
|
|
result.addLengthDistance(length, offset)
|
|
for i in 1..length-1:
|
|
inc pos
|
|
wpos = pos and (nz.windowsize - 1)
|
|
hashval = getHash(nz, insize, pos)
|
|
if usezeros and (hashval == 0):
|
|
if numzeros == 0: numzeros = countZeros(nz, insize, pos)
|
|
elif (pos + numzeros > insize) or (nz.data[pos + numzeros - 1] != chr(0)): dec numzeros
|
|
else: numzeros = 0
|
|
updateHashChain(hash, wpos, hashval, numzeros)
|
|
inc pos
|
|
|
|
proc deflateFixed(nz: nzStream, hash: var NZHash, datapos, dataend: int, final: bool) =
|
|
var tree_ll: HuffmanTree #tree for literal values and length codes
|
|
var tree_d: HuffmanTree #tree for distance codes
|
|
|
|
generateFixedLitLenTree(tree_ll)
|
|
generateFixedDistanceTree(tree_d)
|
|
|
|
nz.bits.addBitToStream(if final: 1 else: 0)
|
|
nz.bits.addBitToStream(1) #first bit of BTYPE
|
|
nz.bits.addBitToStream(0) #second bit of BTYPE
|
|
|
|
if nz.use_lz77: #LZ77 encoded
|
|
var lz77 = nz.encodeLZ77(hash, datapos, dataend)
|
|
nz.bits.writeLZ77data(lz77, tree_ll, tree_d)
|
|
else: #no LZ77, but still will be Huffman compressed
|
|
for i in datapos..dataend-1:
|
|
nz.bits.addHuffmanSymbol(tree_ll, ord(nz.data[i]))
|
|
nz.bits.addHuffmanSymbol(tree_ll, 256) #add END code
|
|
|
|
proc deflateDynamic(nz: nzStream, hash: var NZHash, datapos, dataend: int, final: bool) =
|
|
#A block is compressed as follows: The PNG data is lz77 encoded, resulting in
|
|
#literal bytes and length/distance pairs. This is then huffman compressed with
|
|
#two huffman trees. One huffman tree is used for the lit and len values ("ll"),
|
|
#another huffman tree is used for the dist values ("d"). These two trees are
|
|
#stored using their code lengths, and to compress even more these code lengths
|
|
#are also run-length encoded and huffman compressed. This gives a huffman tree
|
|
#of code lengths "cl". The code lenghts used to describe this third tree are
|
|
#the code length code lengths ("clcl").
|
|
|
|
#The lz77 encoded data, represented with integers
|
|
#since there will also be length and distance codes in it
|
|
|
|
var
|
|
tree_ll: HuffmanTree #tree for lit,len values
|
|
tree_d: HuffmanTree #tree for distance codes
|
|
tree_cl: HuffmanTree #tree for encoding the code lengths representing tree_ll and tree_d
|
|
|
|
frequencies_cl: seq[int] #frequency of code length codes
|
|
bitlen_lld: seq[int] #lit,len,dist code lenghts (int bits), literally (without repeat codes).
|
|
bitlen_lld_e: seq[int] #bitlen_lld encoded with repeat codes (this is a rudemtary run length compression)
|
|
#bitlen_cl is the code length code lengths ("clcl"). The bit lengths of codes to represent tree_cl
|
|
#(these are written as is in the file, it would be crazy to compress these using yet another huffman
|
|
#tree that needs to be represented by yet another set of code lengths)
|
|
bitlen_cl: seq[int]
|
|
datasize = dataend - datapos
|
|
|
|
#Due to the huffman compression of huffman tree representations ("two levels"), there are some anologies:
|
|
#bitlen_lld is to tree_cl what data is to tree_ll and tree_d.
|
|
#bitlen_lld_e is to bitlen_lld what lz77_encoded is to data.
|
|
#bitlen_cl is to bitlen_lld_e what bitlen_lld is to lz77_encoded.
|
|
|
|
var lz77: seq[int]
|
|
|
|
if nz.use_lz77:
|
|
lz77 = nz.encodeLZ77(hash, datapos, dataend)
|
|
else:
|
|
#no LZ77, but still will be Huffman compressed
|
|
lz77 = newSeq[int](datasize)
|
|
for i in datapos..dataend-1: lz77[i] = ord(nz.data[i])
|
|
|
|
var frequencies_ll = newSeq[int](286) #frequency of lit,len codes
|
|
var frequencies_d = newSeq[int](30) #frequency of dist codes
|
|
|
|
#Count the frequencies of lit, len and dist codes
|
|
var i = 0
|
|
while i < lz77.len:
|
|
let symbol = lz77[i]
|
|
inc frequencies_ll[symbol]
|
|
if symbol > 256:
|
|
let dist = lz77[i + 2]
|
|
inc frequencies_d[dist]
|
|
inc(i, 3)
|
|
inc i
|
|
|
|
frequencies_ll[256] = 1 #there will be exactly 1 end code, at the end of the block
|
|
|
|
#Make both huffman trees, one for the lit and len codes, one for the dist codes
|
|
HuffmanTree_makeFromFrequencies(tree_ll, frequencies_ll, 257, 15)
|
|
|
|
#2, not 1, is chosen for mincodes: some buggy PNG decoders require at least 2 symbols in the dist tree
|
|
HuffmanTree_makeFromFrequencies(tree_d, frequencies_d, 2, 15)
|
|
|
|
var numcodes_ll = min(tree_ll.numcodes, 286)
|
|
var numcodes_d = min(tree_d.numcodes, 30)
|
|
|
|
#store the code lengths of both generated trees in bitlen_lld
|
|
bitlen_lld = newSeq[int](numcodes_ll + numcodes_d)
|
|
for i in 0..numcodes_ll-1: bitlen_lld[i] = HuffmanTree_getLength(tree_ll, i)
|
|
for i in 0..numcodes_d-1: bitlen_lld[i+numcodes_ll] = HuffmanTree_getLength(tree_d, i)
|
|
|
|
#run-length compress bitlen_ldd into bitlen_lld_e by using repeat codes 16 (copy length 3-6 times),
|
|
#17 (3-10 zeroes), 18 (11-138 zeroes)
|
|
i = 0
|
|
bitlen_lld_e = @[]
|
|
while i < bitlen_lld.len:
|
|
var j = 0 #amount of repetitions
|
|
while(i + j + 1 < bitlen_lld.len) and (bitlen_lld[i + j + 1] == bitlen_lld[i]): inc j
|
|
|
|
if (bitlen_lld[i] == 0) and (j >= 2): #repeat code for zeroes
|
|
inc j #include the first zero
|
|
if j <= 10: #repeat code 17 supports max 10 zeroes
|
|
bitlen_lld_e.add 17
|
|
bitlen_lld_e.add(j - 3)
|
|
else: #repeat code 18 supports max 138 zeroes
|
|
if j > 138: j = 138
|
|
bitlen_lld_e.add 18
|
|
bitlen_lld_e.add(j - 11)
|
|
i += (j - 1)
|
|
elif j >= 3: #repeat code for value other than zero
|
|
var num = j div 6
|
|
var rest = j mod 6
|
|
bitlen_lld_e.add bitlen_lld[i]
|
|
for k in 0..num-1:
|
|
bitlen_lld_e.add 16
|
|
bitlen_lld_e.add(6 - 3)
|
|
if rest >= 3:
|
|
bitlen_lld_e.add 16
|
|
bitlen_lld_e.add(rest - 3)
|
|
else: j -= rest
|
|
i += j
|
|
else: #too short to benefit from repeat code
|
|
bitlen_lld_e.add bitlen_lld[i]
|
|
inc i
|
|
|
|
#generate tree_cl, the huffmantree of huffmantrees
|
|
frequencies_cl = newSeq[int](NUM_CODE_LENGTH_CODES)
|
|
i = 0
|
|
while i < bitlen_lld_e.len:
|
|
inc frequencies_cl[bitlen_lld_e[i]]
|
|
#after a repeat code come the bits that specify the number of repetitions,
|
|
#those don't need to be in the frequencies_cl calculation
|
|
if bitlen_lld_e[i] >= 16: inc i
|
|
inc i
|
|
|
|
HuffmanTree_makeFromFrequencies(tree_cl, frequencies_cl, frequencies_cl.len, 7)
|
|
|
|
bitlen_cl = newSeq[int](tree_cl.numcodes)
|
|
for i in 0..tree_cl.numcodes-1:
|
|
#lenghts of code length tree is in the order as specified by deflate*/
|
|
bitlen_cl[i] = HuffmanTree_getLength(tree_cl, CLCL_ORDER[i])
|
|
|
|
while(bitlen_cl[bitlen_cl.high] == 0) and (bitlen_cl.len > 4):
|
|
#remove zeros at the end, but minimum size must be 4
|
|
bitlen_cl.setLen(bitlen_cl.high)
|
|
|
|
#Write everything into the output
|
|
#After the BFINAL and BTYPE, the dynamic block consists out of the following:
|
|
#- 5 bits HLIT, 5 bits HDIST, 4 bits HCLEN
|
|
#- (HCLEN+4)*3 bits code lengths of code length alphabet
|
|
#- HLIT + 257 code lenghts of lit/length alphabet (encoded using the code length
|
|
# alphabet, + possible repetition codes 16, 17, 18)
|
|
#- HDIST + 1 code lengths of distance alphabet (encoded using the code length
|
|
# alphabet, + possible repetition codes 16, 17, 18)
|
|
#- compressed data
|
|
#- 256 (end code)
|
|
|
|
#Write block type
|
|
nz.bits.addBitToStream(if final: 1 else: 0)
|
|
nz.bits.addBitToStream(0) #first bit of BTYPE "dynamic"
|
|
nz.bits.addBitToStream(1) #second bit of BTYPE "dynamic"
|
|
|
|
#write the HLIT, HDIST and HCLEN values
|
|
var HLIT = (numcodes_ll - 257)
|
|
var HDIST = (numcodes_d - 1)
|
|
var HCLEN = bitlen_cl.len - 4
|
|
|
|
#trim zeroes for HCLEN. HLIT and HDIST were already trimmed at tree creation
|
|
while(bitlen_cl[HCLEN + 4 - 1] == 0) and (HCLEN > 0): dec HCLEN
|
|
nz.bits.addBitsToStream(HLIT, 5)
|
|
nz.bits.addBitsToStream(HDIST, 5)
|
|
nz.bits.addBitsToStream(HCLEN, 4)
|
|
|
|
#write the code lenghts of the code length alphabet
|
|
for i in 0..HCLEN + 4 - 1: nz.bits.addBitsToStream(bitlen_cl[i], 3)
|
|
|
|
#write the lenghts of the lit/len AND the dist alphabet
|
|
i = 0
|
|
while i < bitlen_lld_e.len:
|
|
nz.bits.addHuffmanSymbol(tree_cl, bitlen_lld_e[i])
|
|
#extra bits of repeat codes
|
|
if bitlen_lld_e[i] == 16:
|
|
inc i
|
|
nz.bits.addBitsToStream(bitlen_lld_e[i], 2)
|
|
elif bitlen_lld_e[i] == 17:
|
|
inc i
|
|
nz.bits.addBitsToStream(bitlen_lld_e[i], 3)
|
|
elif bitlen_lld_e[i] == 18:
|
|
inc i
|
|
nz.bits.addBitsToStream(bitlen_lld_e[i], 7)
|
|
inc i
|
|
|
|
#write the compressed data symbols
|
|
nz.bits.writeLZ77data(lz77, tree_ll, tree_d)
|
|
|
|
if HuffmanTree_getLength(tree_ll, 256) == 0:
|
|
raise newNZError("the length of the end code 256 must be larger than 0")
|
|
|
|
#write the end code
|
|
nz.bits.addHuffmanSymbol(tree_ll, 256)
|
|
|
|
proc nzDeflate(nz: nzStream) =
|
|
var hash: NZHash
|
|
var blocksize = 0
|
|
var insize = nz.data.len
|
|
|
|
if nz.btype > 2: raise newNZError("invalid block type")
|
|
elif nz.btype == 0:
|
|
nz.deflateNoCompression
|
|
return
|
|
elif nz.btype == 1: blocksize = insize
|
|
else: blocksize = max(insize div 8 + 8, 65535) #if(nz.btype == 2)
|
|
#if blocksize < 65535: blocksize = 65535
|
|
|
|
var numdeflateblocks = (insize + blocksize - 1) div blocksize
|
|
if numdeflateblocks == 0: numdeflateblocks = 1
|
|
nimzHashInit(hash, nz.windowsize)
|
|
|
|
for i in 0..numdeflateblocks-1:
|
|
let final = (i == numdeflateblocks - 1)
|
|
let datapos = i * blocksize
|
|
let dataend = min(datapos + blocksize, insize)
|
|
|
|
if nz.btype == 1: nz.deflateFixed(hash, datapos, dataend, final)
|
|
elif nz.btype == 2: nz.deflateDynamic(hash, datapos, dataend, final)
|
|
|
|
proc nzInit(): nzStream =
|
|
const DEFAULT_WINDOWSIZE = 2048
|
|
|
|
result = nzStream(
|
|
#compress with dynamic huffman tree
|
|
#(not in the mathematical sense, just not the predefined one)
|
|
btype : 2,
|
|
use_lz77: true,
|
|
windowsize: DEFAULT_WINDOWSIZE,
|
|
minmatch: 3,
|
|
nicematch: 128,
|
|
lazymatching: true,
|
|
ignoreAdler32: false)
|
|
|
|
proc nzDeflateInit*(input: string): nzStream =
|
|
var nz = nzInit()
|
|
nz.data = input
|
|
nz.bits.data = ""
|
|
nz.bits.bitpointer = 0
|
|
nz.mode = nzsDeflate
|
|
result = nz
|
|
|
|
proc nzInflateInit*(input: string): nzStream =
|
|
var nz = nzInit()
|
|
nz.data = newStringOfCap(1024 * 1024 * 5) # Allocate 5MB in advance
|
|
shallowCopy(nz.bits.data, input)
|
|
nz.bits.bitpointer = 0
|
|
nz.bits.databitlen = input.len * 8
|
|
nz.mode = nzsInflate
|
|
result = nz
|
|
|
|
proc nzGetResult(nz: nzStream): string =
|
|
if nz.mode == nzsInflate: return nz.data
|
|
result = nz.bits.data
|
|
|
|
proc nzAdler32(adler: uint32, data: string): uint32 =
|
|
var s1 = adler and 0xffff
|
|
var s2 = (adler shr 16) and 0xffff
|
|
var len = data.len
|
|
var i = 0
|
|
|
|
while len > 0:
|
|
#at least 5550 sums can be done before the sums overflow
|
|
#saving a lot of module divisions
|
|
|
|
var amount = min(len, 5550)
|
|
dec(len, amount)
|
|
while amount > 0:
|
|
s1 += cast[uint32](ord(data[i]) or 0)
|
|
s2 += s1
|
|
dec(amount)
|
|
inc(i)
|
|
|
|
s1 = s1 mod 65521.uint32
|
|
s2 = s2 mod 65521.uint32
|
|
|
|
result = (s2 shl 16.uint32) or s1
|
|
|
|
proc add32bitInt(s: var BitStream, val: uint32) =
|
|
s.data.add chr(cast[int](val shr 24) and 0xff)
|
|
s.data.add chr(cast[int](val shr 16) and 0xff)
|
|
s.data.add chr(cast[int](val shr 8) and 0xff)
|
|
s.data.add chr(cast[int](val ) and 0xff)
|
|
|
|
proc zlib_compress*(nz: nzStream): string =
|
|
#zlib data: 1 byte CMF (CM+CINFO),
|
|
#1 byte FLG, deflate data,
|
|
#4 byte ADLER32 checksum of the Decompressed data
|
|
|
|
let
|
|
CMF = 120 #0b01111000: CM 8, CINFO 7. With CINFO 7, any window size up to 32768 can be used.
|
|
FLEVEL = 0
|
|
FDICT = 0
|
|
var
|
|
CMFFLG = 256 * CMF + FDICT * 32 + FLEVEL * 64
|
|
FCHECK = 31 - CMFFLG mod 31
|
|
|
|
CMFFLG += FCHECK
|
|
|
|
nz.bits.data.add chr(CMFFLG div 256)
|
|
nz.bits.data.add chr(CMFFLG mod 256)
|
|
nz.bits.bitpointer += 16
|
|
|
|
nz.nzDeflate
|
|
nz.bits.add32bitInt nzAdler32(1, nz.data)
|
|
result = nz.nzGetResult
|
|
|
|
proc readInt32(input: string): uint32 =
|
|
assert input.len == 4
|
|
result = cast[uint32](ord(input[0])) shl 24
|
|
result += cast[uint32](ord(input[1])) shl 16
|
|
result += cast[uint32](ord(input[2])) shl 8
|
|
result += cast[uint32](ord(input[3]))
|
|
|
|
proc zlib_decompress*(nz: nzStream): string =
|
|
let insize = nz.bits.data.len
|
|
|
|
if insize < 2: raise newNZError("size of zlib data too small")
|
|
|
|
#read information from zlib header
|
|
let CMF = nz.bits.readByte
|
|
let FLG = nz.bits.readByte
|
|
|
|
if ((CMF * 256 + FLG) mod 31) != 0:
|
|
raise newNZError(" zlib header must be a multiple of 31")
|
|
#the FCHECK value is supposed to be made that way
|
|
|
|
#let CM = CMF and 15
|
|
#let CINFO = (CMF shr 4) and 15
|
|
#FCHECK = FLG and 31 #FCHECK is already tested above
|
|
#let FDICT = (FLG shr 5) and 1
|
|
#FLEVEL = (FLG shr 6) and 3 #FLEVEL is not used here
|
|
|
|
#if(CM != 8 || CINFO > 7)
|
|
#/*error: only compression method 8: inflate with sliding window of 32k is supported by the PNG spec*/
|
|
#return 25;
|
|
#if(FDICT != 0)
|
|
#/*error: the specification of PNG says about the zlib stream:
|
|
#"The additional flags shall not specify a preset dictionary."*/
|
|
#return 26;
|
|
|
|
let checksum = nz.bits.data.substr(insize-4, insize-1).readInt32
|
|
nz.bits.data.setLen(insize-4)
|
|
|
|
nz.nzInflate
|
|
|
|
if not nz.ignoreAdler32:
|
|
let adler32 = nzAdler32(1, nz.data)
|
|
if checksum != adler32:
|
|
raise newNZError("adler checksum not correct, data must be corrupted")
|
|
|
|
result = nz.nzGetResult
|