internal/binres: table marshal methods with pack func

The OpenTable func now references a prepacked version of the
resources.arsc file with a number of entries removed. The current
size of this file is 62KB. This could be dropped further by
implementing utf8 in string pool during marshal, utf16 encoding exploded
the original size by approximately 20%. Another potential improvement
is to allow type entries to be packed sparsely which may provide
significant savings if not zipping.

Change-Id: Ie139c2bdb0e3c5a9212516d18cf627d75774e187
Reviewed-on: https://go-review.googlesource.com/18649
Reviewed-by: David Crawshaw <crawshaw@golang.org>
This commit is contained in:
Daniel Skinner 2016-01-14 18:06:09 -06:00 committed by David Crawshaw
parent db7fc23f7b
commit b8ddff8878
5 changed files with 546 additions and 35 deletions

View File

@ -50,8 +50,6 @@ import (
"encoding/xml"
"fmt"
"io"
"os"
"path"
"unicode"
)
@ -187,11 +185,7 @@ type xnode struct {
}
func UnmarshalXML(r io.Reader) (*XML, error) {
sdkdir := os.Getenv("ANDROID_HOME")
if sdkdir == "" {
return nil, fmt.Errorf("ANDROID_HOME env var not set")
}
tbl, err := OpenTable(path.Join(sdkdir, "platforms/android-15/android.jar"))
tbl, err := OpenTable()
if err != nil {
return nil, err
}

View File

@ -12,7 +12,7 @@ import (
"log"
"math"
"os"
"path"
"path/filepath"
"strings"
"testing"
)
@ -219,7 +219,7 @@ func TestOpenTable(t *testing.T) {
if sdkdir == "" {
t.Skip("ANDROID_HOME env var not set")
}
tbl, err := OpenTable(path.Join(sdkdir, "platforms/android-15/android.jar"))
tbl, err := OpenTable()
if err != nil {
t.Fatal(err)
}
@ -258,7 +258,7 @@ func TestTableRefByName(t *testing.T) {
if sdkdir == "" {
t.Skip("ANDROID_HOME env var not set")
}
tbl, err := OpenTable(path.Join(sdkdir, "platforms/android-15/android.jar"))
tbl, err := OpenTable()
if err != nil {
t.Fatal(err)
}
@ -276,6 +276,132 @@ func TestTableRefByName(t *testing.T) {
}
}
func testPackResources(t *testing.T) {
packResources()
f, err := os.Open(filepath.Join("data", "packed.arsc.gz"))
if err != nil {
t.Fatal(err)
}
fi, err := f.Stat()
if err != nil {
t.Fatal(err)
}
t.Logf("packed.arsc.gz %vKB", fi.Size()/1024)
}
func TestTableMarshal(t *testing.T) {
tbl, err := OpenSDKTable()
if err != nil {
t.Fatal(err)
}
bin, err := tbl.MarshalBinary()
if err != nil {
t.Fatal(err)
}
xtbl := new(Table)
if err := xtbl.UnmarshalBinary(bin); err != nil {
t.Fatal(err)
}
if len(tbl.pool.strings) != len(xtbl.pool.strings) {
t.Fatal("tbl.pool lengths don't match")
}
if len(tbl.pkgs) != len(xtbl.pkgs) {
t.Fatal("tbl.pkgs lengths don't match")
}
pkg, xpkg := tbl.pkgs[0], xtbl.pkgs[0]
if err := compareStrings(t, pkg.typePool.strings, xpkg.typePool.strings); err != nil {
t.Fatal(err)
}
if err := compareStrings(t, pkg.keyPool.strings, xpkg.keyPool.strings); err != nil {
t.Fatal(err)
}
if len(pkg.specs) != len(xpkg.specs) {
t.Fatal("pkg.specs lengths don't match")
}
for i, spec := range pkg.specs {
xspec := xpkg.specs[i]
if spec.id != xspec.id {
t.Fatal("spec.id doesn't match")
}
if spec.entryCount != xspec.entryCount {
t.Fatal("spec.entryCount doesn't match")
}
if len(spec.entries) != len(xspec.entries) {
t.Fatal("spec.entries lengths don't match")
}
for j, mask := range spec.entries {
xmask := xspec.entries[j]
if mask != xmask {
t.Fatal("entry mask doesn't match")
}
}
if len(spec.types) != len(xspec.types) {
t.Fatal("spec.types length don't match")
}
for j, typ := range spec.types {
xtyp := xspec.types[j]
if typ.id != xtyp.id {
t.Fatal("typ.id doesn't match")
}
if typ.entryCount != xtyp.entryCount {
t.Fatal("typ.entryCount doesn't match")
}
if typ.entriesStart != xtyp.entriesStart {
t.Fatal("typ.entriesStart doesn't match")
}
if len(typ.indices) != len(xtyp.indices) {
t.Fatal("typ.indices length don't match")
}
for k, index := range typ.indices {
xindex := xtyp.indices[k]
if index != xindex {
t.Errorf("type index doesn't match at %v, have %v, want %v", k, xindex, index)
}
}
if len(typ.entries) != len(xtyp.entries) {
t.Fatal("typ.entries lengths don't match")
}
for k, nt := range typ.entries {
xnt := xtyp.entries[k]
if nt == nil {
if xnt != nil {
t.Fatal("nt is nil but xnt is not")
}
continue
}
if nt.size != xnt.size {
t.Fatal("entry.size doesn't match")
}
if nt.flags != xnt.flags {
t.Fatal("entry.flags don't match")
}
if nt.key != xnt.key {
t.Fatal("entry.key doesn't match")
}
if nt.parent != xnt.parent {
t.Fatal("entry.parent doesn't match")
}
if nt.count != xnt.count {
t.Fatal("entry.count doesn't match")
}
for l, val := range nt.values {
xval := xnt.values[l]
if val.name != xval.name {
t.Fatal("value.name doesn't match")
}
}
}
}
}
}
func BenchmarkTableRefByName(b *testing.B) {
sdkdir := os.Getenv("ANDROID_HOME")
if sdkdir == "" {
@ -285,7 +411,7 @@ func BenchmarkTableRefByName(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
tbl, err := OpenTable(path.Join(sdkdir, "platforms/android-15/android.jar"))
tbl, err := OpenTable()
if err != nil {
b.Fatal(err)
}

Binary file not shown.

142
internal/binres/sdk.go Normal file
View File

@ -0,0 +1,142 @@
package binres
import (
"archive/zip"
"bytes"
"compress/gzip"
"fmt"
"io"
"os"
"path"
"path/filepath"
)
func apiJarPath() (string, error) {
sdkdir := os.Getenv("ANDROID_HOME")
if sdkdir == "" {
return "", fmt.Errorf("ANDROID_HOME env var not set")
}
return path.Join(sdkdir, "platforms/android-15/android.jar"), nil
}
func apiResources() ([]byte, error) {
p, err := apiJarPath()
if err != nil {
return nil, err
}
zr, err := zip.OpenReader(p)
if err != nil {
return nil, err
}
defer zr.Close()
buf := new(bytes.Buffer)
for _, f := range zr.File {
if f.Name == "resources.arsc" {
rc, err := f.Open()
if err != nil {
return nil, err
}
_, err = io.Copy(buf, rc)
if err != nil {
return nil, err
}
rc.Close()
break
}
}
if buf.Len() == 0 {
return nil, fmt.Errorf("failed to read resources.arsc")
}
return buf.Bytes(), nil
}
// packResources produces a stripped down version of the resources.arsc from api jar.
func packResources() error {
tbl, err := OpenSDKTable()
if err != nil {
return err
}
tbl.pool.strings = []string{} // should not be needed
pkg := tbl.pkgs[0]
// drop language string entries
for _, typ := range pkg.specs[3].types {
if typ.config.locale.language != 0 {
for j, nt := range typ.entries {
if nt == nil { // NoEntry
continue
}
pkg.keyPool.strings[nt.key] = ""
typ.indices[j] = NoEntry
typ.entries[j] = nil
}
}
}
// drop strings from pool for specs to be dropped
for _, spec := range pkg.specs[4:] {
for _, typ := range spec.types {
for _, nt := range typ.entries {
if nt == nil { // NoEntry
continue
}
// don't drop if there's a collision
var collision bool
for _, xspec := range pkg.specs[:4] {
for _, xtyp := range xspec.types {
for _, xnt := range xtyp.entries {
if xnt == nil {
continue
}
if collision = nt.key == xnt.key; collision {
break
}
}
}
}
if !collision {
pkg.keyPool.strings[nt.key] = ""
}
}
}
}
// entries are densely packed but probably safe to drop nil entries off the end
for _, spec := range pkg.specs[:4] {
for _, typ := range spec.types {
var last int
for i, nt := range typ.entries {
if nt != nil {
last = i
}
}
typ.entries = typ.entries[:last+1]
typ.indices = typ.indices[:last+1]
}
}
// keeping 0:attr, 1:id, 2:style, 3:string
pkg.typePool.strings = pkg.typePool.strings[:4]
pkg.specs = pkg.specs[:4]
bin, err := tbl.MarshalBinary()
if err != nil {
return err
}
f, err := os.Create(filepath.Join("data", "packed.arsc.gz"))
if err != nil {
return err
}
defer f.Close()
zw := gzip.NewWriter(f)
defer zw.Close()
if _, err := zw.Write(bin); err != nil {
return err
}
return nil
}

View File

@ -5,11 +5,13 @@
package binres
import (
"archive/zip"
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"unicode/utf16"
)
@ -56,33 +58,36 @@ type Table struct {
pkgs []*Package
}
// OpenTable decodes resources.arsc from an sdk platform jar.
func OpenTable(path string) (*Table, error) {
zr, err := zip.OpenReader(path)
// OpenSDKTable decodes resources.arsc from sdk platform jar.
func OpenSDKTable() (*Table, error) {
bin, err := apiResources()
if err != nil {
return nil, err
}
tbl := new(Table)
if err := tbl.UnmarshalBinary(bin); err != nil {
return nil, err
}
return tbl, nil
}
// OpenTable decodes the prepacked resources.arsc for the supported sdk platform.
func OpenTable() (*Table, error) {
f, err := os.Open(filepath.Join("data", "packed.arsc.gz"))
if err != nil {
return nil, err
}
defer f.Close()
zr, err := gzip.NewReader(f)
if err != nil {
return nil, err
}
defer zr.Close()
buf := new(bytes.Buffer)
for _, f := range zr.File {
if f.Name == "resources.arsc" {
rc, err := f.Open()
if err != nil {
return nil, err
}
_, err = io.Copy(buf, rc)
if err != nil {
return nil, err
}
rc.Close()
break
}
var buf bytes.Buffer
if _, err := io.Copy(&buf, zr); err != nil {
return nil, err
}
if buf.Len() == 0 {
return nil, fmt.Errorf("failed to read resources.arsc")
}
tbl := new(Table)
if err := tbl.UnmarshalBinary(buf.Bytes()); err != nil {
return nil, err
@ -167,6 +172,38 @@ func (tbl *Table) UnmarshalBinary(bin []byte) error {
return nil
}
func (tbl *Table) MarshalBinary() ([]byte, error) {
bin := make([]byte, 12)
putu16(bin, uint16(ResTable))
putu16(bin[2:], 12)
putu32(bin[8:], uint32(len(tbl.pkgs)))
if tbl.pool.IsUTF8() {
tbl.pool.flags ^= UTF8Flag
defer func() {
tbl.pool.flags |= UTF8Flag
}()
}
b, err := tbl.pool.MarshalBinary()
if err != nil {
return nil, err
}
bin = append(bin, b...)
for _, pkg := range tbl.pkgs {
b, err = pkg.MarshalBinary()
if err != nil {
return nil, err
}
bin = append(bin, b...)
}
putu32(bin[4:], uint32(len(bin)))
return bin, nil
}
// Package contains a collection of resource data types.
type Package struct {
chunkHeader
@ -257,6 +294,62 @@ func (pkg *Package) UnmarshalBinary(bin []byte) error {
return nil
}
func (pkg *Package) MarshalBinary() ([]byte, error) {
bin := make([]byte, 284)
putu16(bin, uint16(ResTablePackage))
putu16(bin[2:], 284)
putu32(bin[8:], pkg.id)
p := utf16.Encode([]rune(pkg.name))
for i, x := range p {
putu16(bin[12+i*2:], x)
}
if pkg.typePool != nil {
if pkg.typePool.IsUTF8() {
pkg.typePool.flags ^= UTF8Flag
defer func() {
pkg.typePool.flags |= UTF8Flag
}()
}
b, err := pkg.typePool.MarshalBinary()
if err != nil {
return nil, err
}
putu32(bin[268:], uint32(len(bin)))
putu32(bin[272:], pkg.lastPublicType)
bin = append(bin, b...)
}
if pkg.keyPool != nil {
if pkg.keyPool.IsUTF8() {
pkg.keyPool.flags ^= UTF8Flag
defer func() {
pkg.keyPool.flags |= UTF8Flag
}()
}
b, err := pkg.keyPool.MarshalBinary()
if err != nil {
return nil, err
}
putu32(bin[276:], uint32(len(bin)))
putu32(bin[280:], pkg.lastPublicKey)
bin = append(bin, b...)
}
for _, spec := range pkg.specs {
b, err := spec.MarshalBinary()
if err != nil {
return nil, err
}
bin = append(bin, b...)
}
putu32(bin[4:], uint32(len(bin)))
return bin, nil
}
// TypeSpec provides a specification for the resources defined by a particular type.
type TypeSpec struct {
chunkHeader
@ -289,6 +382,31 @@ func (spec *TypeSpec) UnmarshalBinary(bin []byte) error {
return nil
}
func (spec *TypeSpec) MarshalBinary() ([]byte, error) {
bin := make([]byte, 16+len(spec.entries)*4)
putu16(bin, uint16(ResTableTypeSpec))
putu16(bin[2:], 16)
putu32(bin[4:], uint32(len(bin)))
bin[8] = byte(spec.id)
// [9] = 0
// [10:12] = 0
putu32(bin[12:], uint32(len(spec.entries)))
for i, x := range spec.entries {
putu32(bin[16+i*4:], x)
}
for _, typ := range spec.types {
b, err := typ.MarshalBinary()
if err != nil {
return nil, err
}
bin = append(bin, b...)
}
return bin, nil
}
// Type provides a collection of entries for a specific device configuration.
type Type struct {
chunkHeader
@ -298,8 +416,46 @@ type Type struct {
entryCount uint32 // number of uint32 entry configuration masks that follow
entriesStart uint32 // offset from header where Entry data starts
// TODO implement and decode Config if strictly necessary
// config Config // configuration this collection of entries is designed for (portrait, sw600dp, etc)
// configuration this collection of entries is designed for
config struct {
size uint32
imsi struct {
mcc uint16 // mobile country code
mnc uint16 // mobile network code
}
locale struct {
language uint16
country uint16
}
screenType struct {
orientation uint8
touchscreen uint8
density uint16
}
input struct {
keyboard uint8
navigation uint8
inputFlags uint8
inputPad0 uint8
}
screenSize struct {
width uint16
height uint16
}
version struct {
sdk uint16
minor uint16 // always 0
}
screenConfig struct {
layout uint8
uiMode uint8
smallestWidthDP uint16
}
screenSizeDP struct {
width uint16
height uint16
}
}
indices []uint32 // values that map to typePool
entries []*Entry
@ -323,6 +479,30 @@ func (typ *Type) UnmarshalBinary(bin []byte) error {
return fmt.Errorf("res0 res1 not zero")
}
typ.config.size = btou32(bin[20:])
typ.config.imsi.mcc = btou16(bin[24:])
typ.config.imsi.mnc = btou16(bin[26:])
typ.config.locale.language = btou16(bin[28:])
typ.config.locale.country = btou16(bin[30:])
typ.config.screenType.orientation = uint8(bin[32])
typ.config.screenType.touchscreen = uint8(bin[33])
typ.config.screenType.density = btou16(bin[34:])
typ.config.input.keyboard = uint8(bin[36])
typ.config.input.navigation = uint8(bin[37])
typ.config.input.inputFlags = uint8(bin[38])
typ.config.input.inputPad0 = uint8(bin[39])
typ.config.screenSize.width = btou16(bin[40:])
typ.config.screenSize.height = btou16(bin[42:])
typ.config.version.sdk = btou16(bin[44:])
typ.config.version.minor = btou16(bin[46:])
typ.config.screenConfig.layout = uint8(bin[48])
typ.config.screenConfig.uiMode = uint8(bin[49])
typ.config.screenConfig.smallestWidthDP = btou16(bin[50:])
typ.config.screenSizeDP.width = btou16(bin[52:])
typ.config.screenSizeDP.height = btou16(bin[54:])
// fmt.Println("language/country:", u16tos(typ.config.locale.language), u16tos(typ.config.locale.country))
buf := bin[typ.headerByteSize:typ.entriesStart]
for len(buf) > 0 {
typ.indices = append(typ.indices, btou32(buf))
@ -348,6 +528,36 @@ func (typ *Type) UnmarshalBinary(bin []byte) error {
return nil
}
func (typ *Type) MarshalBinary() ([]byte, error) {
bin := make([]byte, 56+len(typ.entries)*4)
putu16(bin, uint16(ResTableType))
putu16(bin[2:], 56)
bin[8] = byte(typ.id)
// [9] = 0
// [10:12] = 0
putu32(bin[12:], uint32(len(typ.entries)))
putu32(bin[16:], uint32(56+len(typ.entries)*4))
var ntbin []byte
for i, nt := range typ.entries {
if nt == nil { // NoEntry
putu32(bin[56+i*4:], NoEntry)
continue
}
putu32(bin[56+i*4:], uint32(len(ntbin)))
b, err := nt.MarshalBinary()
if err != nil {
return nil, err
}
ntbin = append(ntbin, b...)
}
bin = append(bin, ntbin...)
putu32(bin[4:], uint32(len(bin)))
return bin, nil
}
// Entry is a resource key typically followed by a value or resource map.
type Entry struct {
size uint16
@ -389,6 +599,34 @@ func (nt *Entry) UnmarshalBinary(bin []byte) error {
return nil
}
func (nt *Entry) MarshalBinary() ([]byte, error) {
bin := make([]byte, 8)
putu16(bin, nt.size)
putu16(bin[2:], nt.flags)
putu32(bin[4:], uint32(nt.key))
if nt.size == 16 {
bin = append(bin, make([]byte, 8+len(nt.values)*12)...)
putu32(bin[8:], uint32(nt.parent))
putu32(bin[12:], uint32(len(nt.values)))
for i, val := range nt.values {
b, err := val.MarshalBinary()
if err != nil {
return nil, err
}
copy(bin[16+i*12:], b)
}
} else {
b, err := nt.values[0].data.MarshalBinary()
if err != nil {
return nil, err
}
bin = append(bin, b...)
}
return bin, nil
}
type Value struct {
name TableRef
data *Data
@ -400,6 +638,17 @@ func (val *Value) UnmarshalBinary(bin []byte) error {
return val.data.UnmarshalBinary(bin[4:])
}
func (val *Value) MarshalBinary() ([]byte, error) {
bin := make([]byte, 12)
putu32(bin, uint32(val.name))
b, err := val.data.MarshalBinary()
if err != nil {
return nil, err
}
copy(bin[4:], b)
return bin, nil
}
type DataType uint8
// explicitly defined for clarity and resolvability with apt source