gc:arc refactor 'filterScanline' and add tests

This commit is contained in:
andri lim 2020-04-13 18:36:40 +07:00
parent 27e76f2d65
commit 43d5548237
No known key found for this signature in database
GPG Key ID: 31702AE10541E6B9
4 changed files with 468 additions and 0 deletions

View File

@ -13,10 +13,13 @@ task tests, "Run tests":
exec "nim c -r tests/test_codec.nim" exec "nim c -r tests/test_codec.nim"
exec "nim c -r tests/test_suite.nim" exec "nim c -r tests/test_suite.nim"
exec "nim c -r tests/test_nimz.nim" exec "nim c -r tests/test_nimz.nim"
exec "nim c -r tests/test_filters.nim"
exec "nim c -r -d:release tests/test_apng.nim" exec "nim c -r -d:release tests/test_apng.nim"
exec "nim c -r -d:release tests/test_codec.nim" exec "nim c -r -d:release tests/test_codec.nim"
exec "nim c -r -d:release tests/test_suite.nim" exec "nim c -r -d:release tests/test_suite.nim"
exec "nim c -r -d:release tests/test_nimz.nim" exec "nim c -r -d:release tests/test_nimz.nim"
exec "nim c -r -d:release tests/test_filters.nim"
exec "nim c -r --gc:arc -d:release tests/test_nimz.nim" exec "nim c -r --gc:arc -d:release tests/test_nimz.nim"
exec "nim c -r --gc:arc -d:release tests/test_filters.nim"

310
nimPNG/filters.nim Normal file
View File

@ -0,0 +1,310 @@
type
PNGFilter* = enum
FLT_NONE,
FLT_SUB,
FLT_UP,
FLT_AVERAGE,
FLT_PAETH
# Paeth predicter, used by PNG filter type 4
proc paethPredictor(a, b, c: int): uint =
let pa = abs(b - c)
let pb = abs(a - c)
let pc = abs(a + b - c - c)
if(pc < pa) and (pc < pb): return c.uint
elif pb < pa: return b.uint
result = a.uint
proc filterScanline*(output: var openArray[byte], input: openArray[byte], byteWidth, len: int, filterType: PNGFilter) =
template currPix: untyped = input[i].uint
template prevPix: untyped = input[i - byteWidth].uint
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = byte((currPix - prevPix) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = input[i]
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = byte((currPix - (prevPix div 2)) and 0xFF)
of FLT_PAETH:
for i in 0..<byteWidth:
output[i] = input[i]
#paethPredictor(prevPix, 0, 0) is always prevPix
for i in byteWidth..<len:
output[i] = byte((currPix - prevPix) and 0xFF)
proc filterScanline*(output: var openArray[byte], input, prevLine: openArray[byte], byteWidth, len: int, filterType: PNGFilter) =
template currPix: untyped = input[i].uint
template prevPix: untyped = input[i - byteWidth].uint
template upPix: untyped = prevLine[i].uint
template prevPixI: untyped = input[i - byteWidth].int
template upPixI: untyped = prevLine[i].int
template prevUpPix: untyped = prevLine[i - byteWidth].int
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = byte((currPix - prevPix) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = byte((currPix - upPix) and 0xFF)
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = byte((currPix - (upPix div 2)) and 0xFF)
for i in byteWidth..<len:
output[i] = byte((currPix - ((prevPix + upPix) div 2)) and 0xFF)
of FLT_PAETH:
#paethPredictor(0, upPix, 0) is always upPix
for i in 0..<byteWidth:
output[i] = byte((currPix - upPix) and 0xFF)
for i in byteWidth..<len:
output[i] = byte((currPix - paethPredictor(prevPixI, upPixI, prevUpPix)) and 0xFF)
#[
proc filterZero(output: var DataBuf, input: DataBuf, w, h, bpp: int) =
#the width of a input in bytes, not including the filter type
let lineBytes = (w * bpp + 7) div 8
#byteWidth is used for filtering, is 1 when bpp < 8, number of bytes per pixel otherwise
let byteWidth = (bpp + 7) div 8
var prevLine: DataBuf
for y in 0..h-1:
let outindex = (1 + lineBytes) * y #the extra filterbyte added to each row
let inindex = lineBytes * y
output[outindex] = byte(int(FLT_NONE)) #filter type byte
var outp = output.subbuffer(outindex + 1)
let input = input.subbuffer(inindex)
filterScanline(outp, input, prevLine, lineBytes, byteWidth, FLT_NONE)
prevLine = input.subbuffer(inindex)
proc filterMinsum(output: var DataBuf, input: DataBuf, w, h, bpp: int) =
let lineBytes = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
#adaptive filtering
var sum = [0, 0, 0, 0, 0]
var smallest = 0
#five filtering attempts, one for each filter type
var attempt: array[0..4, string]
var bestType = 0
var prevLine: DataBuf
for i in 0..attempt.high:
attempt[i] = newString(lineBytes)
for y in 0..h-1:
#try the 5 filter types
for fType in 0..4:
var outp = initBuffer(attempt[fType])
filterScanline(outp, input.subbuffer(y * lineBytes), prevLine, lineBytes, byteWidth, PNGFilter0(fType))
#calculate the sum of the result
sum[fType] = 0
if fType == 0:
for x in 0..lineBytes-1:
sum[fType] += ord(attempt[fType][x])
else:
for x in 0..lineBytes-1:
#For differences, each byte should be treated as signed, values above 127 are negative
#(converted to signed char). Filtertype 0 isn't a difference though, so use unsigned there.
#This means filtertype 0 is almost never chosen, but that is justified.
let s = ord(attempt[fType][x])
if s < 128: sum[fType] += s
else: sum[fType] += (255 - s)
#check if this is smallest sum (or if type == 0 it's the first case so always store the values)*/
if(fType == 0) or (sum[fType] < smallest):
bestType = fType
smallest = sum[fType]
prevLine = input.subbuffer(y * lineBytes)
#now fill the out values
#the first byte of a input will be the filter type
output[y * (lineBytes + 1)] = byte(bestType)
for x in 0..lineBytes-1:
output[y * (lineBytes + 1) + 1 + x] = attempt[bestType][x]
proc filterEntropy(output: var DataBuf, input: DataBuf, w, h, bpp: int) =
let lineBytes = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
var prevLine: DataBuf
var sum: array[0..4, float]
var smallest = 0.0
var bestType = 0
var attempt: array[0..4, string]
var count: array[0..255, int]
for i in 0..attempt.high:
attempt[i] = newString(lineBytes)
for y in 0..h-1:
#try the 5 filter types
for fType in 0..4:
var outp = initBuffer(attempt[fType])
filterScanline(outp, input.subbuffer(y * lineBytes), prevLine, lineBytes, byteWidth, PNGFilter0(fType))
for x in 0..255: count[x] = 0
for x in 0..lineBytes-1:
inc count[ord(attempt[fType][x])]
inc count[fType] #the filter type itself is part of the input
sum[fType] = 0
for x in 0..255:
let p = float(count[x]) / float(lineBytes + 1)
if count[x] != 0: sum[fType] += log2(1 / p) * p
#check if this is smallest sum (or if type == 0 it's the first case so always store the values)
if (fType == 0) or (sum[fType] < smallest):
bestType = fType
smallest = sum[fType]
prevLine = input.subbuffer(y * lineBytes)
#now fill the out values*/
#the first byte of a input will be the filter type
output[y * (lineBytes + 1)] = byte(bestType)
for x in 0..lineBytes-1:
output[y * (lineBytes + 1) + 1 + x] = attempt[bestType][x]
proc filterPredefined(output: var DataBuf, input: DataBuf, w, h, bpp: int, state: PNGEncoder) =
let lineBytes = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
var prevLine: DataBuf
for y in 0..h-1:
let outindex = (1 + lineBytes) * y #the extra filterbyte added to each row
let inindex = lineBytes * y
let fType = ord(state.predefinedFilters[y])
output[outindex] = byte(fType) #filter type byte
var outp = output.subbuffer(outindex + 1)
filterScanline(outp, input.subbuffer(inindex), prevLine, lineBytes, byteWidth, PNGFilter0(fType))
prevLine = input.subbuffer(inindex)
proc filterBruteForce(output: var DataBuf, input: DataBuf, w, h, bpp: int) =
let lineBytes = (w * bpp + 7) div 8
let byteWidth = (bpp + 7) div 8
var prevLine: DataBuf
#brute force filter chooser.
#deflate the input after every filter attempt to see which one deflates best.
#This is very slow and gives only slightly smaller, sometimes even larger, result*/
var size: array[0..4, int]
var attempt: array[0..4, string] #five filtering attempts, one for each filter type
var smallest = 0
var bestType = 0
#use fixed tree on the attempts so that the tree is not adapted to the filtertype on purpose,
#to simulate the true case where the tree is the same for the whole image. Sometimes it gives
#better result with dynamic tree anyway. Using the fixed tree sometimes gives worse, but in rare
#cases better compression. It does make this a bit less slow, so it's worth doing this.
for i in 0..attempt.high:
attempt[i] = newString(lineBytes)
for y in 0..h-1:
#try the 5 filter types
for fType in 0..4:
#let testSize = attempt[fType].len
var outp = initBuffer(attempt[fType])
filterScanline(outp, input.subbuffer(y * lineBytes), prevLine, lineBytes, byteWidth, PNGFilter0(fType))
size[fType] = 0
var nz = nzDeflateInit(attempt[fType])
let data = zlib_compress(nz)
size[fType] = data.len
#check if this is smallest size (or if type == 0 it's the first case so always store the values)
if(fType == 0) or (size[fType] < smallest):
bestType = fType
smallest = size[fType]
prevLine = input.subbuffer(y * lineBytes)
output[y * (lineBytes + 1)] = byte(bestType) #the first byte of a input will be the filter type
for x in 0..lineBytes-1:
output[y * (lineBytes + 1) + 1 + x] = attempt[bestType][x]
]#
proc unfilterScanline*(output: var openArray[byte], input: openArray[byte], byteWidth, len: int, filterType: PNGFilter) =
# When the pixels are smaller than 1 byte, the filter works byte per byte (byteWidth = 1)
# the incoming inputs do NOT include the filtertype byte, that one is given in the parameter filterType instead
# output and input MAY be the same memory address! output must be disjoint.
template currPix: untyped = input[i].uint
template prevPix: untyped = output[i - byteWidth].uint
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = byte((currPix + prevPix) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = input[i]
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = byte((currPix + (prevPix div 2)) and 0xFF)
of FLT_PAETH:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
# paethPredictor(prevPix, 0, 0) is always prevPix
output[i] = byte((currPix + prevPix) and 0xFF)
proc unfilterScanline*(output: var openArray[byte], input, prevLine: openArray[byte], byteWidth, len: int, filterType: PNGFilter) =
# For PNG filter method 0
# unfilter a PNG image input by input. when the pixels are smaller than 1 byte,
# the filter works byte per byte (byteWidth = 1)
# prevLine is the previous unfiltered input, output the result, input the current one
# the incoming inputs do NOT include the filtertype byte, that one is given in the parameter filterType instead
# output and input MAY be the same memory address! prevLine must be disjoint.
template currPix: untyped = input[i].uint
template prevPix: untyped = output[i - byteWidth].uint
template upPix: untyped = prevLine[i].uint
template prevPixI: untyped = output[i - byteWidth].int
template upPixI: untyped = prevLine[i].int
template prevUpPix: untyped = prevLine[i - byteWidth].int
case filterType
of FLT_NONE:
for i in 0..<len:
output[i] = input[i]
of FLT_SUB:
for i in 0..<byteWidth:
output[i] = input[i]
for i in byteWidth..<len:
output[i] = byte((currPix + prevPix) and 0xFF)
of FLT_UP:
for i in 0..<len:
output[i] = byte((currPix + upPix) and 0xFF)
of FLT_AVERAGE:
for i in 0..<byteWidth:
output[i] = byte((currPix + upPix div 2) and 0xFF)
for i in byteWidth..<len:
output[i] = byte((currPix + ((prevPix + upPix) div 2)) and 0xFF)
of FLT_PAETH:
for i in 0..<byteWidth:
# paethPredictor(0, upPix, 0) is always upPix
output[i] = byte((currPix + upPix) and 0xFF)
for i in byteWidth..<len:
output[i] = byte((currPix + paethPredictor(prevPixI, upPixI, prevUpPix)) and 0xFF)

58
tests/randutils.nim Normal file
View File

@ -0,0 +1,58 @@
import random, sets
type
RandGen*[T] = object
minVal, maxVal: T
Bytes* = seq[byte]
proc rng*[T](minVal, maxVal: T): RandGen[T] =
doAssert(minVal <= maxVal)
result.minVal = minVal
result.maxVal = maxVal
proc rng*[T](minMax: T): RandGen[T] =
rng(minMax, minMax)
proc getVal*[T](x: RandGen[T]): T =
if x.minVal == x.maxVal: return x.minVal
rand(x.minVal..x.maxVal)
proc randString*(len: int): string =
result = newString(len)
for i in 0..<len:
result[i] = rand(255).char
proc randBytes*(len: int): Bytes =
result = newSeq[byte](len)
for i in 0..<len:
result[i] = rand(255).byte
proc randPrimitives*[T](val: int): T =
when T is string:
randString(val)
elif T is int:
result = val
elif T is byte:
result = val.byte
elif T is Bytes:
result = randBytes(val)
proc randList*(T: typedesc, fillGen: RandGen, listLen: int, unique: static[bool] = true): seq[T] =
result = newSeqOfCap[T](listLen)
when unique:
var set = initSet[T]()
for len in 0..<listLen:
while true:
let x = randPrimitives[T](fillGen.getVal())
if x notin set:
result.add x
set.incl x
break
else:
for len in 0..<listLen:
let x = randPrimitives[T](fillGen.getVal())
result.add x
proc randList*(T: typedesc, fillGen, listGen: RandGen, unique: static[bool] = true): seq[T] =
randList(T, fillGen, listGen.getVal(), unique)

97
tests/test_filters.nim Normal file
View File

@ -0,0 +1,97 @@
import ../nimPNG/filters, ./randutils, unittest
template roundTripFilter(byteWidth: int, filter: PNGFilter) =
filterScanline(outPix, inPix, byteWidth, lineBytes, filter)
unfilterScanline(oriPix, outPix, byteWidth, lineBytes, filter)
check oriPix == inPix
template roundTripFilterP(byteWidth: int, filter: PNGFilter) =
# with prevLine
filterScanline(outPix, inPix, prevLine, byteWidth, lineBytes, filter)
unfilterScanline(oriPix, outPix, prevLine, byteWidth, lineBytes, filter)
check oriPix == inPix
proc testFilterScanline() =
suite "filterScanline":
const
lineBytes = 128
let
inPix = randList(byte, rng(0, 255), lineBytes, unique = false)
prevLine = randList(byte, rng(0, 255), lineBytes, unique = false)
var
outPix = newSeq[byte](lineBytes)
oriPix = newSeq[byte](lineBytes)
test "FLT_NONE":
roundTripFilter(1, FLT_NONE)
roundTripFilter(2, FLT_NONE)
roundTripFilter(3, FLT_NONE)
roundTripFilter(4, FLT_NONE)
test "FLT_SUB":
roundTripFilter(1, FLT_SUB)
roundTripFilter(2, FLT_SUB)
roundTripFilter(3, FLT_SUB)
roundTripFilter(4, FLT_SUB)
test "FLT_UP":
roundTripFilter(1, FLT_UP)
roundTripFilter(2, FLT_UP)
roundTripFilter(3, FLT_UP)
roundTripFilter(4, FLT_UP)
test "FLT_AVERAGE":
roundTripFilter(1, FLT_AVERAGE)
roundTripFilter(2, FLT_AVERAGE)
roundTripFilter(3, FLT_AVERAGE)
roundTripFilter(4, FLT_AVERAGE)
test "FLT_PAETH":
roundTripFilter(1, FLT_PAETH)
roundTripFilter(2, FLT_PAETH)
roundTripFilter(3, FLT_PAETH)
roundTripFilter(4, FLT_PAETH)
test "FLT_NONE with prevLine":
roundTripFilterP(1, FLT_NONE)
roundTripFilterP(2, FLT_NONE)
roundTripFilterP(3, FLT_NONE)
roundTripFilterP(4, FLT_NONE)
test "FLT_SUB with prevLine":
roundTripFilterP(1, FLT_SUB)
roundTripFilterP(2, FLT_SUB)
roundTripFilterP(3, FLT_SUB)
roundTripFilterP(4, FLT_SUB)
test "FLT_UP with prevLine":
roundTripFilterP(1, FLT_UP)
roundTripFilterP(2, FLT_UP)
roundTripFilterP(3, FLT_UP)
roundTripFilterP(4, FLT_UP)
test "FLT_AVERAGE with prevLine":
roundTripFilterP(1, FLT_AVERAGE)
roundTripFilterP(2, FLT_AVERAGE)
roundTripFilterP(3, FLT_AVERAGE)
roundTripFilterP(4, FLT_AVERAGE)
test "FLT_PAETH with prevLine":
roundTripFilterP(1, FLT_PAETH)
roundTripFilterP(2, FLT_PAETH)
roundTripFilterP(3, FLT_PAETH)
roundTripFilterP(4, FLT_PAETH)
#of LFS_ZERO: filterZero(output, input, w, h, bpp)
#of LFS_MINSUM: filterMinsum(output, input, w, h, bpp)
#of LFS_ENTROPY: filterEntropy(output, input, w, h, bpp)
#of LFS_BRUTE_FORCE: filterBruteForce(output, input, w, h, bpp)
#of LFS_PREDEFINED: filterPredefined(output, input, w, h, bpp, state)
proc main() =
testFilterScanline()
main()