Merge pull request #40 from jangko/gc_arc

gc:arc refactor
This commit is contained in:
andri lim 2020-04-14 15:45:31 +07:00 committed by GitHub
commit 676bca486a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1194 additions and 787 deletions

1042
nimPNG.nim

File diff suppressed because it is too large Load Diff

View File

@ -9,14 +9,6 @@ skipDirs = @["tests", "docs"]
requires "nim >= 0.19.0"
task tests, "Run tests":
exec "nim c -r tests/test_apng.nim"
exec "nim c -r tests/test_codec.nim"
exec "nim c -r tests/test_suite.nim"
exec "nim c -r tests/test_nimz.nim"
exec "nim c -r -d:release tests/test_apng.nim"
exec "nim c -r -d:release tests/test_codec.nim"
exec "nim c -r -d:release tests/test_suite.nim"
exec "nim c -r -d:release tests/test_nimz.nim"
exec "nim c -r --gc:arc -d:release tests/test_nimz.nim"
exec "nim -v"
exec "nim c -r -d:release tests/all_tests"
exec "nim c -r --gc:arc -d:release tests/all_tests"

585
nimPNG/filters.nim Normal file
View File

@ -0,0 +1,585 @@
import math, ../nimPNG/nimz
type
PNGFilter* = enum
FLT_NONE,
FLT_SUB,
FLT_UP,
FLT_AVERAGE,
FLT_PAETH
PNGPass* = object
w*, h*: array[0..6, int]
filterStart*, paddedStart*, start*: array[0..7, int]
const
# shared values used by multiple Adam7 related functions
ADAM7_IX* = [ 0, 4, 0, 2, 0, 1, 0 ] # x start values
ADAM7_IY* = [ 0, 0, 4, 0, 2, 0, 1 ] # y start values
ADAM7_DX* = [ 8, 8, 4, 4, 2, 2, 1 ] # x delta values
ADAM7_DY* = [ 8, 8, 8, 4, 4, 2, 2 ] # y delta values
# Paeth predicter, used by PNG filter type 4
proc paethPredictor(a, b, c: int): uint =
let pa = abs(b - c)
let pb = abs(a - c)
let pc = abs(a + b - c - c)
if(pc < pa) and (pc < pb): return c.uint
elif pb < pa: return b.uint
result = a.uint
proc filterScanline*[T](output: var openArray[T], input: openArray[T], byteWidth, len: int, filterType: PNGFilter) =
template currPix(i): untyped = input[i].uint
template prevPix(i): untyped = input[i - byteWidth].uint
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = T((currPix(i) - prevPix(i)) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = input[i]
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = T((currPix(i) - (prevPix(i) div 2)) and 0xFF)
of FLT_PAETH:
for i in 0..<byteWidth:
output[i] = input[i]
# paethPredictor(prevPix, 0, 0) is always prevPix
for i in byteWidth..<len:
output[i] = T((currPix(i) - prevPix(i)) and 0xFF)
proc filterScanline*[T](output: var openArray[T], input, prevLine: openArray[T], byteWidth, len: int, filterType: PNGFilter) =
template currPix(i): untyped = input[i].uint
template prevPix(i): untyped = input[i - byteWidth].uint
template upPix(i): untyped = prevLine[i].uint
template prevPixI(i): untyped = input[i - byteWidth].int
template upPixI(i): untyped = prevLine[i].int
template prevUpPix(i): untyped = prevLine[i - byteWidth].int
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = T((currPix(i) - prevPix(i)) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = T((currPix(i) - upPix(i)) and 0xFF)
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = T((currPix(i) - (upPix(i) div 2)) and 0xFF)
for i in byteWidth..<len:
output[i] = T((currPix(i) - ((prevPix(i) + upPix(i)) div 2)) and 0xFF)
of FLT_PAETH:
# paethPredictor(0, upPix, 0) is always upPix
for i in 0..<byteWidth:
output[i] = T((currPix(i) - upPix(i)) and 0xFF)
for i in byteWidth..<len:
output[i] = T((currPix(i) - paethPredictor(prevPixI(i), upPixI(i), prevUpPix(i))) and 0xFF)
proc filterZero*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
# the width of a input in Ts, not including the filter type
let lineTs = (w * bpp + 7) div 8
# byteWidth is used for filtering, is 1 when bpp < 8, number of Ts per pixel otherwise
let byteWidth = (bpp + 7) div 8
# line 0
if h > 0:
output[0] = T(FLT_NONE) # filterType T
filterScanline(output.toOpenArray(1, output.len-1), # skip filterType
input, byteWidth, lineTs, FLT_NONE)
# next line start from 1
var prevIndex = 0
for y in 1..<h:
let outIndex = (1 + lineTs) * y # the extra filterType added to each row
let inIndex = lineTs * y
output[outIndex] = T(FLT_NONE) # filterType T
filterScanline(output.toOpenArray(outIndex + 1, output.len-1), # skip filterType
input.toOpenArray(inIndex, input.len-1),
input.toOpenArray(prevIndex, input.len-1),
byteWidth, lineTs, FLT_NONE)
prevIndex = inIndex
proc filterMinsum*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
let lineTs = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
#adaptive filtering
var
sum = [0, 0, 0, 0, 0]
smallest = 0
# five filtering attempts, one for each filter type
attempt: array[0..4, seq[T]]
bestType = 0
prevIndex = 0
for i in 0..attempt.high:
attempt[i] = newSeq[T](lineTs)
for y in 0..<h:
# try the 5 filter types
let inIndex = y * lineTs
for fType in 0..4:
if y == 0:
filterScanline(attempt[fType],
input.toOpenArray(inIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
else:
filterScanline(attempt[fType],
input.toOpenArray(inIndex, input.len-1),
input.toOpenArray(prevIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
# calculate the sum of the result
sum[fType] = 0
if fType == 0:
for x in 0..lineTs-1:
sum[fType] += int(attempt[fType][x])
else:
for x in 0..lineTs-1:
# For differences, each T should be treated as signed, values above 127 are negative
# (converted to signed char). Filtertype 0 isn't a difference though, so use unsigned there.
# This means filtertype 0 is almost never chosen, but that is justified.
let s = int(attempt[fType][x])
if s < 128: sum[fType] += s
else: sum[fType] += (255 - s)
# check if this is smallest sum (or if type == 0 it's the first case so always store the values)
if(fType == 0) or (sum[fType] < smallest):
bestType = fType
smallest = sum[fType]
prevIndex = inIndex
# now fill the out values
# the first T of a input will be the filter type
output[y * (lineTs + 1)] = T(bestType)
for x in 0..lineTs-1:
output[y * (lineTs + 1) + 1 + x] = attempt[bestType][x]
proc filterEntropy*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
let lineTs = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
var
sum: array[0..4, float]
smallest = 0.0
bestType = 0
attempt: array[0..4, seq[T]]
count: array[0..255, int]
prevIndex = 0
for i in 0..attempt.high:
attempt[i] = newSeq[T](lineTs)
for y in 0..<h:
# try the 5 filter types
let inIndex = y * lineTs
for fType in 0..4:
if y == 0:
filterScanline(attempt[fType],
input.toOpenArray(inIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
else:
filterScanline(attempt[fType],
input.toOpenArray(inIndex, input.len-1),
input.toOpenArray(prevIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
for x in 0..255: count[x] = 0
for x in 0..lineTs-1:
inc count[int(attempt[fType][x])]
inc count[fType] # the filterType itself is part of the input
sum[fType] = 0
for x in 0..255:
let p = float(count[x]) / float(lineTs + 1)
if count[x] != 0: sum[fType] += log2(1 / p) * p
# check if this is smallest sum (or if type == 0 it's the first case so always store the values)
if (fType == 0) or (sum[fType] < smallest):
bestType = fType
smallest = sum[fType]
prevIndex = inIndex
# now fill the out values
# the first T of a input will be the filter type
output[y * (lineTs + 1)] = T(bestType)
for x in 0..<lineTs:
output[y * (lineTs + 1) + 1 + x] = attempt[bestType][x]
proc filterPredefined*[T](output: var openArray[T], input: openArray[T],
w, h, bpp: int, predefinedFilters: openArray[PNGFilter]) =
let lineTs = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
# line 0
if h > 0:
output[0] = T(predefinedFilters[0]) # filterType T
filterScanline(output.toOpenArray(1, output.len-1), # skip filterType
input, byteWidth, lineTs, predefinedFilters[0])
# next line start from 1
var prevIndex = 0
for y in 1..<h:
let outIndex = (1 + lineTs) * y # the extra filterType added to each row
let inIndex = lineTs * y
let fType = ord(predefinedFilters[y])
output[outIndex] = T(fType) # filterType T
filterScanline(output.toOpenArray(outIndex + 1, output.len-1), # skip filterType
input.toOpenArray(inIndex, input.len-1),
input.toOpenArray(prevIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
prevIndex = inIndex
proc filterBruteForce*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
let lineTs = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
# brute force filter chooser.
# deflate the input after every filter attempt to see which one deflates best.
# This is very slow and gives only slightly smaller, sometimes even larger, result*/
var
size: array[0..4, int]
# five filtering attempts, one for each filter type
attempt: array[0..4, seq[T]]
smallest = 0
bestType = 0
prevIndex = 0
# use fixed tree on the attempts so that the tree is not adapted to the filtertype on purpose,
# to simulate the true case where the tree is the same for the whole image. Sometimes it gives
# better result with dynamic tree anyway. Using the fixed tree sometimes gives worse, but in rare
# cases better compression. It does make this a bit less slow, so it's worth doing this.
for i in 0..attempt.high:
attempt[i] = newSeq[T](lineTs)
for y in 0..h-1:
# try the 5 filter types
let inIndex = y * lineTs
for fType in 0..4:
if y == 0:
filterScanline(attempt[fType],
input.toOpenArray(inIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
else:
filterScanline(attempt[fType],
input.toOpenArray(inIndex, input.len-1),
input.toOpenArray(prevIndex, input.len-1),
byteWidth, lineTs, PNGFilter(fType))
size[fType] = 0
var nz = nzCompressInit(attempt[fType])
let data = zlib_compress(nz)
size[fType] = data.len
#check if this is smallest size (or if type == 0 it's the first case so always store the values)
if(fType == 0) or (size[fType] < smallest):
bestType = fType
smallest = size[fType]
prevIndex = inIndex
output[y * (lineTs + 1)] = T(bestType) # the first T of a input will be the filter type
for x in 0..lineTs-1:
output[y * (lineTs + 1) + 1 + x] = attempt[bestType][x]
proc unfilterScanline*[T](output: var openArray[T], input: openArray[T], byteWidth, len: int, filterType: PNGFilter) =
# When the pixels are smaller than 1 T, the filter works T per T (byteWidth = 1)
# the incoming inputs do NOT include the filtertype T, that one is given in the parameter filterType instead
# output and input MAY be the same memory address! output must be disjoint.
template currPix(i): untyped = input[i].uint
template prevPix(i): untyped = output[i - byteWidth].uint
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = T((currPix(i) + prevPix(i)) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = input[i]
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = T((currPix(i) + (prevPix(i) div 2)) and 0xFF)
of FLT_PAETH:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
# paethPredictor(prevPix, 0, 0) is always prevPix
output[i] = T((currPix(i) + prevPix(i)) and 0xFF)
proc unfilterScanline*[T](output: var openArray[T], input, prevLine: openArray[T], byteWidth, len: int, filterType: PNGFilter) =
# For PNG filter method 0
# unfilter a PNG image input by input. when the pixels are smaller than 1 T,
# the filter works T per T (byteWidth = 1)
# prevLine is the previous unfiltered input, output the result, input the current one
# the incoming inputs do NOT include the filtertype T, that one is given in the parameter filterType instead
# output and input MAY be the same memory address! prevLine must be disjoint.
template currPix(i): untyped = input[i].uint
template prevPix(i): untyped = output[i - byteWidth].uint
template upPix(i): untyped = prevLine[i].uint
template prevPixI(i): untyped = output[i - byteWidth].int
template upPixI(i): untyped = prevLine[i].int
template prevUpPix(i): untyped = prevLine[i - byteWidth].int
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = T((currPix(i) + prevPix(i)) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = T((currPix(i) + upPix(i)) and 0xFF)
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = T((currPix(i) + upPix(i) div 2) and 0xFF)
for i in byteWidth..<len:
output[i] = T((currPix(i) + ((prevPix(i) + upPix(i)) div 2)) and 0xFF)
of FLT_PAETH:
for i in 0..<byteWidth:
# paethPredictor(0, upPix, 0) is always upPix
output[i] = T((currPix(i) + upPix(i)) and 0xFF)
for i in byteWidth..<len:
output[i] = T((currPix(i) + paethPredictor(prevPixI(i), upPixI(i), prevUpPix(i))) and 0xFF)
proc unfilter*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
# For PNG filter method 0
# this function unfilters a single image (e.g. without interlacing this is called once, with Adam7 seven times)
# output must have enough Ts allocated already, input must have the scanLines + 1 filtertype T per scanLine
# w and h are image dimensions or dimensions of reduced image, bpp is bits per pixel
# input and output are allowed to be the same memory address (but aren't the same size since in has the extra filter Ts)
# byteWidth is used for filtering, is 1 when bpp < 8, number of Ts per pixel otherwise
let byteWidth = (bpp + 7) div 8
let lineTs = (w * bpp + 7) div 8
# line 0, without prevLine
if h > 0:
unfilterScanLine(output,
input.toOpenArray(1, input.len-1), # skip the filterType
byteWidth, lineTs,
PNGFilter(input[0]))
# next line start from 1
var prevIndex = 0
for y in 1..<h:
let outIndex = lineTs * y
let inIndex = (1 + lineTs) * y # the extra filterT added to each row
let filterType = PNGFilter(input[inIndex])
unfilterScanLine(output.toOpenArray(outIndex, output.len-1),
input.toOpenArray(inIndex + 1, input.len-1), # skip the filterType
output.toOpenArray(prevIndex, output.len-1), # prevLine
byteWidth, lineTs, filterType)
prevIndex = outIndex
proc readBitFromReversedStream*[T](bitptr: var int, bitstream: openArray[T]): int =
result = ((int(bitstream[bitptr shr 3]) shr (7 - (bitptr and 0x7))) and 1)
inc bitptr
proc readBitsFromReversedStream*[T](bitptr: var int, bitstream: openArray[T], nbits: int): int =
result = 0
var i = nbits - 1
while i > -1:
result += readBitFromReversedStream(bitptr, bitstream) shl i
dec i
proc `&=`[T](a: var T, b: T) =
a = T(int(a) and int(b))
proc `|=`[T](a: var T, b: T) =
a = T(int(a) or int(b))
proc setBitOfReversedStream0*[T](bitptr: var int, bitstream: var openArray[T], bit: int) =
# the current bit in bitstream must be 0 for this to work
if bit != 0:
# earlier bit of huffman code is in a lesser significant bit of an earlier T
bitstream[bitptr shr 3] |= cast[T](bit shl (7 - (bitptr and 0x7)))
inc bitptr
proc setBitOfReversedStream*[T](bitptr: var int, bitstream: var openArray[T], bit: int) =
# the current bit in bitstream may be 0 or 1 for this to work
if bit == 0: bitstream[bitptr shr 3] &= cast[T](not (1 shl (7 - (bitptr and 0x7))))
else: bitstream[bitptr shr 3] |= cast[T](1 shl (7 - (bitptr and 0x7)))
inc bitptr
proc removePaddingBits*[T](output: var openArray[T], input: openArray[T], olinebits, ilinebits, h: int) =
# After filtering there are still padding bits if scanLines have non multiple of 8 bit amounts. They need
# to be removed (except at last scanLine of (Adam7-reduced) image) before working with pure image buffers
# for the Adam7 code, the color convert code and the output to the user.
# in and out are allowed to be the same buffer, in may also be higher but still overlapping; in must
# have >= ilinebits*h bits, out must have >= olinebits*h bits, olinebits must be <= ilinebits
# also used to move bits after earlier such operations happened, e.g. in a sequence of reduced images from Adam7
# only useful if (ilinebits - olinebits) is a value in the range 1..7
let diff = ilinebits - olinebits
var
ibp = 0
obp = 0 # input and output bit pointers
for y in 0..h-1:
for x in 0..olinebits-1:
var bit = readBitFromReversedStream(ibp, input)
setBitOfReversedStream(obp, output, bit)
inc(ibp, diff)
# Outputs various dimensions and positions in the image related to the Adam7 reduced images.
# passw: output containing the width of the 7 passes
# passh: output containing the height of the 7 passes
# filter_passstart: output containing the index of the start and end of each
# reduced image with filter Ts
# padded_passstart output containing the index of the start and end of each
# reduced image when without filter Ts but with padded scanLines
# passstart: output containing the index of the start and end of each reduced
# image without padding between scanLines, but still padding between the images
# w, h: width and height of non-interlaced image
# bpp: bits per pixel
# "padded" is only relevant if bpp is less than 8 and a scanLine or image does not
# end at a full T
proc adam7PassValues*(pass: var PNGPass, w, h, bpp: int) =
# the passstart values have 8 values:
# the 8th one indicates the T after the end of the 7th (= last) pass
# calculate width and height in pixels of each pass
for i in 0..6:
pass.w[i] = (w + ADAM7_DX[i] - ADAM7_IX[i] - 1) div ADAM7_DX[i]
pass.h[i] = (h + ADAM7_DY[i] - ADAM7_IY[i] - 1) div ADAM7_DY[i]
if pass.w[i] == 0: pass.h[i] = 0
if pass.h[i] == 0: pass.w[i] = 0
pass.filterStart[0] = 0
pass.paddedStart[0] = 0
pass.start[0] = 0
for i in 0..6:
# if passw[i] is 0, it's 0 Ts, not 1 (no filtertype-T)
pass.filterStart[i + 1] = pass.filterStart[i]
if (pass.w[i] != 0) and (pass.h[i] != 0):
pass.filterStart[i + 1] += pass.h[i] * (1 + (pass.w[i] * bpp + 7) div 8)
# bits padded if needed to fill full T at end of each scanLine
pass.paddedStart[i + 1] = pass.paddedStart[i] + pass.h[i] * ((pass.w[i] * bpp + 7) div 8)
# only padded at end of reduced image
pass.start[i + 1] = pass.start[i] + (pass.h[i] * pass.w[i] * bpp + 7) div 8
# input: Adam7 interlaced image, with no padding bits between scanLines, but between
# reduced images so that each reduced image starts at a T.
# output: the same pixels, but re-ordered so that they're now a non-interlaced image with size w*h
# bpp: bits per pixel
# output has the following size in bits: w * h * bpp.
# input is possibly bigger due to padding bits between reduced images.
# output must be big enough AND must be 0 everywhere if bpp < 8 in the current implementation
# (because that's likely a little bit faster)
# NOTE: comments about padding bits are only relevant if bpp < 8
proc adam7Deinterlace*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
var pass: PNGPass
adam7PassValues(pass, w, h, bpp)
if bpp >= 8:
for i in 0..6:
let byteWidth = bpp div 8
for y in 0..<pass.h[i]:
for x in 0..<pass.w[i]:
let inStart = pass.start[i] + (y * pass.w[i] + x) * byteWidth
let outStart = ((ADAM7_IY[i] + y * ADAM7_DY[i]) * w + ADAM7_IX[i] + x * ADAM7_DX[i]) * byteWidth
for b in 0..<byteWidth:
output[outStart + b] = input[inStart + b]
else: # bpp < 8: Adam7 with pixels < 8 bit is a bit trickier: with bit pointers
for i in 0..6:
let ilinebits = bpp * pass.w[i]
let olinebits = bpp * w
for y in 0..<pass.h[i]:
for x in 0..<pass.w[i]:
var ibp = (8 * pass.start[i]) + (y * ilinebits + x * bpp)
var obp = (ADAM7_IY[i] + y * ADAM7_DY[i]) * olinebits + (ADAM7_IX[i] + x * ADAM7_DX[i]) * bpp
for b in 0..<bpp:
let bit = readBitFromReversedStream(ibp, input)
# note that this function assumes the out buffer is completely 0, use setBitOfReversedStream otherwise
setBitOfReversedStream0(obp, output, bit)
# input: non-interlaced image with size w*h
# output: the same pixels, but re-ordered according to PNG's Adam7 interlacing, with
# no padding bits between scanlines, but between reduced images so that each
# reduced image starts at a T.
# bpp: bits per pixel
# there are no padding bits, not between scanlines, not between reduced images
# in has the following size in bits: w * h * bpp.
# output is possibly bigger due to padding bits between reduced images
# NOTE: comments about padding bits are only relevant if bpp < 8
proc adam7Interlace*[T](output: var openArray[T], input: openArray[T], w, h, bpp: int) =
var pass: PNGPass
adam7PassValues(pass, w, h, bpp)
if bpp >= 8:
for i in 0..6:
let byteWidth = bpp div 8
for y in 0..<pass.h[i]:
for x in 0..<pass.w[i]:
let inStart = ((ADAM7_IY[i] + y * ADAM7_DY[i]) * w + ADAM7_IX[i] + x * ADAM7_DX[i]) * byteWidth
let outStart = pass.start[i] + (y * pass.w[i] + x) * byteWidth
for b in 0..<byteWidth:
output[outStart + b] = input[inStart + b]
else: # bpp < 8: Adam7 with pixels < 8 bit is a bit trickier: with bit pointers
for i in 0..6:
let ilinebits = bpp * pass.w[i]
let olinebits = bpp * w
for y in 0..<pass.h[i]:
for x in 0..<pass.w[i]:
var ibp = (ADAM7_IY[i] + y * ADAM7_DY[i]) * olinebits + (ADAM7_IX[i] + x * ADAM7_DX[i]) * bpp
var obp = (8 * pass.start[i]) + (y * ilinebits + x * bpp)
for b in 0..<bpp:
let bit = readBitFromReversedStream(ibp, input)
setBitOfReversedStream(obp, output, bit)
# index: bitgroup index, bits: bitgroup size(1, 2 or 4), in: bitgroup value, out: octet array to add bits to
proc addColorBits*[T](output: var openArray[T], index, bits, input: int) =
var m = 1
if bits == 1: m = 7
elif bits == 2: m = 3
# p = the partial index in the byte, e.g. with 4 palettebits it is 0 for first half or 1 for second half
let p = index and m
var val = input and ((1 shl bits) - 1) # filter out any other bits of the input value
val = val shl (bits * (m - p))
let idx = index * bits div 8
if p == 0: output[idx] = T(val)
else: output[idx] = T(int(output[idx]) or val)
proc addPaddingBits*[T](output: var openArray[T], input: openArray[T], olinebits, ilinebits, h: int) =
#The opposite of the removePaddingBits function
#olinebits must be >= ilinebits
let diff = olinebits - ilinebits
var
obp = 0
ibp = 0 #bit pointers
for y in 0..h-1:
for x in 0..ilinebits-1:
let bit = readBitFromReversedStream(ibp, input)
setBitOfReversedStream(obp, output, bit)
for x in 0..diff-1: setBitOfReversedStream(obp, output, 0)

View File

@ -1,4 +1,4 @@
import streams, sequtils, algorithm, strutils
import sequtils
const
FIRST_LENGTH_CODE_INDEX = 257
@ -1224,6 +1224,12 @@ proc nzDeflateInit*(input: string): nzStream =
template nzCompressInit*(input: string): nzStream =
nzDeflateInit(input)
template nzCompressInit*(input: seq[byte]): nzStream =
nzDeflateInit(cast[string](input))
template nzCompressInit*(input: seq[char]): nzStream =
nzDeflateInit(cast[string](input))
proc nzInflateInit*(input: string): nzStream =
var nz = nzInit()
nz.data = newStringOfCap(1024 * 1024 * 5) # Allocate 5MB in advance

14
nimPNG/png_types.nim Normal file
View File

@ -0,0 +1,14 @@
type
PNGFilterStrategy* = enum
#every filter at zero
LFS_ZERO,
#Use filter that gives minimum sum, as described in the official PNG filter heuristic.
LFS_MINSUM,
#Use the filter type that gives smallest Shannon entropy for this scanLine. Depending
#on the image, this is better or worse than minsum.
LFS_ENTROPY,
#Brute-force-search PNG filters by compressing each filter for each scanLine.
#Experimental, very slow, and only rarely gives better compression than MINSUM.
LFS_BRUTE_FORCE,
#use predefined_filters buffer: you specify the filter type for each scanLine
LFS_PREDEFINED

6
tests/all_tests.nim Normal file
View File

@ -0,0 +1,6 @@
import
test_apng,
test_codec,
test_suite,
test_nimz,
test_filters

61
tests/randutils.nim Normal file
View File

@ -0,0 +1,61 @@
import random, sets
type
RandGen*[T] = object
minVal, maxVal: T
Bytes* = seq[byte]
proc rng*[T](minVal, maxVal: T): RandGen[T] =
doAssert(minVal <= maxVal)
result.minVal = minVal
result.maxVal = maxVal
proc rng*[T](minMax: T): RandGen[T] =
rng(minMax, minMax)
proc getVal*[T](x: RandGen[T]): T =
if x.minVal == x.maxVal: return x.minVal
rand(x.minVal..x.maxVal)
proc randString*(len: int): string =
result = newString(len)
for i in 0..<len:
result[i] = rand(255).char
proc randBytes*(len: int): Bytes =
result = newSeq[byte](len)
for i in 0..<len:
result[i] = rand(255).byte
proc randPrimitives*[T](val: int): T =
type
ByteLike = uint8 | byte | char
when T is string:
randString(val)
elif T is int:
result = val
elif T is ByteLike:
result = T(val)
elif T is Bytes:
result = randBytes(val)
proc randList*(T: typedesc, fillGen: RandGen, listLen: int, unique: static[bool] = true): seq[T] =
result = newSeqOfCap[T](listLen)
when unique:
var set = initHashSet[T]()
for len in 0..<listLen:
while true:
let x = randPrimitives[T](fillGen.getVal())
if x notin set:
result.add x
set.incl x
break
else:
for len in 0..<listLen:
let x = randPrimitives[T](fillGen.getVal())
result.add x
proc randList*(T: typedesc, fillGen, listGen: RandGen, unique: static[bool] = true): seq[T] =
randList(T, fillGen, listGen.getVal(), unique)

View File

@ -1,5 +1,5 @@
import ../nimPNG, streams, math, strutils, tables, base64, os
import ../nimPNG/buffer
import ../nimPNG/[buffer, filters]
type
Image = ref object
@ -67,9 +67,15 @@ proc doCodecTest(image: Image, state: PNGEncoder) =
#if image.data.len > 512:
#assertTrue(s.data.len < image.data.len, "compressed size")
#debugEcho "PNG LEN: ", s.getPosition()
#debugEcho "PNG DATA: ", s.data.toHex
s.setPosition 0
var decoded = s.decodePNG(image.colorType, image.bitDepth)
#debugEcho "DECODED LEN: ", decoded.data.len
#debugEcho "DECODED DATA: ", decoded.data.toHex
assertEquals(image.width, decoded.width)
assertEquals(image.height, decoded.height)
@ -485,12 +491,12 @@ proc testPredefinedFilters() =
var state = makePNGEncoder()
state.filterStrategy = LFS_PREDEFINED
state.filterPaletteZero = false
# everything to filter type 'FLT_AVERAGE'
state.predefinedFilters = newSeq[PNGFilter](h)
for i in 0..<h:
state.predefinedFilters[i] = FLT_AVERAGE
var png = encodePNG(image.data, w, h, state)
var outFilters = png.getFilterTypes()
@ -591,12 +597,15 @@ proc colorConvertTest(bits_in: string, colorType_in: PNGcolorType, bitDepth_in:
echo "color convert test ", bits_in, " - ", bits_out
let expected = bitStringToBytes(bits_out)
let image = initBuffer(bitStringToBytes(bits_in))
let image = bitStringToBytes(bits_in)
let modeIn = newColorMode(colorType_in, bitDepth_in)
let modeOut = newColorMode(colorType_out, bitDepth_out)
var actual = newString(expected.len)
var actualView = initBuffer(actual)
convert(actualView, image, modeOut, modeIn, 1)
convert(
actual.toOpenArray(0, actual.len-1),
image.toOpenArray(0, image.len-1),
modeOut, modeIn, 1)
for i in 0..expected.high:
assertEquals(expected[i].int, actual[i].int, "byte " & $i)
@ -674,19 +683,19 @@ proc testColorConvert2() =
(colorType: LCT_RGBA, bitDepth: 8),
(colorType: LCT_RGBA, bitDepth: 16)]
eight = initBuffer([0,0,0,255, 255,255,255,255,
eight = [0,0,0,255, 255,255,255,255,
0,0,0,255, 255,255,255,255,
255,255,255,255, 0,0,0,255,
255,255,255,255, 255,255,255,255,
0,0,0,255].toString()) #input in RGBA8
0,0,0,255].toString() #input in RGBA8
var
modeIn = newColorMode()
modeOut = newColorMode()
mode_8 = newColorMode()
input = initBuffer(newString(72))
output = initBuffer(newString(72))
eight2 = initBuffer(newString(36))
input = newString(72)
output = newString(72)
eight2 = newString(36)
for i in 0..255:
let j = if i == 1: 255 else: i
@ -701,10 +710,10 @@ proc testColorConvert2() =
modeOut.colorType = cmb.colorType
modeOut.bitDepth = cmb.bitDepth
convert(input, eight, modeIn, mode_8, 3 * 3)
convert(output, input, modeOut, modeIn, 3 * 3) #Test input to output type
convert(eight2, output, mode_8, modeOut, 3 * 3)
assertEquals(eight.data, eight2.data)
convert(input.toOpenArray(0, input.len-1), eight.toOpenArray(0, eight.len-1), modeIn, mode_8, 3 * 3)
convert(output.toOpenArray(0, output.len-1), input.toOpenArray(0, input.len-1), modeOut, modeIn, 3 * 3) #Test input to output type
convert(eight2.toOpenArray(0, eight2.len-1), output.toOpenArray(0, output.len-1), mode_8, modeOut, 3 * 3)
assertEquals(eight, eight2)
#tests that there are no crashes with auto color chooser in case of palettes with translucency etc...
proc testPaletteToPaletteConvert() =
@ -1198,5 +1207,5 @@ proc doMain() =
testNoAutoConvert()
testAutoColorModels()
testFilter()
doMain()

179
tests/test_filters.nim Normal file
View File

@ -0,0 +1,179 @@
import ../nimPNG/filters, ./randutils, unittest, typetraits
template roundTripFilter(byteWidth: int, filter: PNGFilter) =
filterScanline(outPix, inPix, byteWidth, lineBytes, filter)
unfilterScanline(oriPix, outPix, byteWidth, lineBytes, filter)
check oriPix == inPix
template roundTripFilterP(byteWidth: int, filter: PNGFilter) =
# with prevLine
filterScanline(outPix, inPix, prevLine, byteWidth, lineBytes, filter)
unfilterScanline(oriPix, outPix, prevLine, byteWidth, lineBytes, filter)
check oriPix == inPix
proc testFilterScanline(TT: type) =
suite "filterScanline " & TT.name:
const
lineBytes = 128
let
inPix = randList(TT, rng(0, 255), lineBytes, unique = false)
prevLine = randList(TT, rng(0, 255), lineBytes, unique = false)
var
outPix = newSeq[TT](lineBytes)
oriPix = newSeq[TT](lineBytes)
test "FLT_NONE":
roundTripFilter(1, FLT_NONE)
roundTripFilter(2, FLT_NONE)
roundTripFilter(3, FLT_NONE)
roundTripFilter(4, FLT_NONE)
test "FLT_SUB":
roundTripFilter(1, FLT_SUB)
roundTripFilter(2, FLT_SUB)
roundTripFilter(3, FLT_SUB)
roundTripFilter(4, FLT_SUB)
test "FLT_UP":
roundTripFilter(1, FLT_UP)
roundTripFilter(2, FLT_UP)
roundTripFilter(3, FLT_UP)
roundTripFilter(4, FLT_UP)
test "FLT_AVERAGE":
roundTripFilter(1, FLT_AVERAGE)
roundTripFilter(2, FLT_AVERAGE)
roundTripFilter(3, FLT_AVERAGE)
roundTripFilter(4, FLT_AVERAGE)
test "FLT_PAETH":
roundTripFilter(1, FLT_PAETH)
roundTripFilter(2, FLT_PAETH)
roundTripFilter(3, FLT_PAETH)
roundTripFilter(4, FLT_PAETH)
test "FLT_NONE with prevLine":
roundTripFilterP(1, FLT_NONE)
roundTripFilterP(2, FLT_NONE)
roundTripFilterP(3, FLT_NONE)
roundTripFilterP(4, FLT_NONE)
test "FLT_SUB with prevLine":
roundTripFilterP(1, FLT_SUB)
roundTripFilterP(2, FLT_SUB)
roundTripFilterP(3, FLT_SUB)
roundTripFilterP(4, FLT_SUB)
test "FLT_UP with prevLine":
roundTripFilterP(1, FLT_UP)
roundTripFilterP(2, FLT_UP)
roundTripFilterP(3, FLT_UP)
roundTripFilterP(4, FLT_UP)
test "FLT_AVERAGE with prevLine":
roundTripFilterP(1, FLT_AVERAGE)
roundTripFilterP(2, FLT_AVERAGE)
roundTripFilterP(3, FLT_AVERAGE)
roundTripFilterP(4, FLT_AVERAGE)
test "FLT_PAETH with prevLine":
roundTripFilterP(1, FLT_PAETH)
roundTripFilterP(2, FLT_PAETH)
roundTripFilterP(3, FLT_PAETH)
roundTripFilterP(4, FLT_PAETH)
proc checkPixels[T](a, b: openArray[T], len: int): bool =
result = true
for x in 0..<len:
if a[x] != b[x]:
return false
template roundTripStrategy(bpp: int, strategy: untyped) =
block:
let
lineBytes = (w * bpp + 7) div 8
numBytes = h * lineBytes
strategy(outPix, inPix, w, h, bpp)
unfilter(oriPix, outPix, w, h, bpp)
check checkPixels(inPix, oriPix, numBytes)
template roundTripZero(bpp: int) =
roundTripStrategy(bpp, filterZero)
template roundTripEntropy(bpp: int) =
roundTripStrategy(bpp, filterEntropy)
template roundTripMinsum(bpp: int) =
roundTripStrategy(bpp, filterMinsum)
template roundTripBruteforce(bpp: int) =
roundTripStrategy(bpp, filterBruteforce)
template roundTripPredefined(bpp: int) =
block:
let
lineBytes = (w * bpp + 7) div 8
numBytes = h * lineBytes
filterPredefined(outPix, inPix, w, h, bpp, predefinedFilters)
unfilter(oriPix, outPix, w, h, bpp)
check checkPixels(inPix, oriPix, numBytes)
proc testFilterStrategies(TT: type) =
suite "Filter Strategies " & TT.name:
let
h = 128
w = 128
bpp = 32 # we use largest bpp to avoid reallocation
lineBytes = (w * bpp + 7) div 8
numBytes = h * lineBytes
inPix = randList(TT, rng(0, 255), numBytes, unique = false)
outBytes = h * (lineBytes + 1) # lineBytes + filterType
byteFilter = randList(TT, rng(0, 4), h, unique = false)
var
outPix = newSeq[TT](outBytes)
oriPix = newSeq[TT](numBytes)
test "LFS_ZERO":
roundTripZero(8)
roundTripZero(16)
roundTripZero(24)
roundTripZero(32)
test "LFS_PREDEFINED":
var predefinedFilters = newSeq[PNG_FILTER](h)
for i in 0..<h: predefinedFilters[i] = byteFilter[i].PNGFilter
roundTripPredefined(8)
roundTripPredefined(16)
roundTripPredefined(24)
roundTripPredefined(32)
test "LFS_ENTROPY":
roundTripEntropy(8)
roundTripEntropy(16)
roundTripEntropy(24)
roundTripEntropy(32)
test "LFS_MINSUM":
roundTripMinsum(8)
roundTripMinsum(16)
roundTripMinsum(24)
roundTripMinsum(32)
test "LFS_BRUTEFORCE":
roundTripBruteforce(8)
roundTripBruteforce(16)
roundTripBruteforce(24)
roundTripBruteforce(32)
proc main() =
testFilterScanline(byte)
testFilterScanline(char)
testFilterStrategies(byte)
testFilterStrategies(char)
main()

View File

@ -12,17 +12,20 @@ template check_roundtrip(source) =
if uncomp != input:
check false
suite "nimz":
check_roundtrip("alice29.txt")
check_roundtrip("house.jpg")
check_roundtrip("html")
check_roundtrip("urls.10K")
check_roundtrip("fireworks.jpeg")
check_roundtrip("paper-100k.pdf")
check_roundtrip("html_x_4")
check_roundtrip("asyoulik.txt")
check_roundtrip("lcet10.txt")
check_roundtrip("plrabn12.txt")
check_roundtrip("geo.protodata")
check_roundtrip("kppkn.gtb")
check_roundtrip("Mark.Twain-Tom.Sawyer.txt")
proc main() =
suite "nimz":
check_roundtrip("alice29.txt")
check_roundtrip("house.jpg")
check_roundtrip("html")
check_roundtrip("urls.10K")
check_roundtrip("fireworks.jpeg")
check_roundtrip("paper-100k.pdf")
check_roundtrip("html_x_4")
check_roundtrip("asyoulik.txt")
check_roundtrip("lcet10.txt")
check_roundtrip("plrabn12.txt")
check_roundtrip("geo.protodata")
check_roundtrip("kppkn.gtb")
check_roundtrip("Mark.Twain-Tom.Sawyer.txt")
main()