cmd/gomobile: new binary resource implementation

Change-Id: I9708170ac2c5914bb8e978b4703f4e6f7415c737
Reviewed-on: https://go-review.googlesource.com/16553
Reviewed-by: David Crawshaw <crawshaw@golang.org>

internal/binres/binres.go

@@ -0,0 +1,342 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate stringer -output binres_string.go -type ResType,DataType
+
+// Package binres implements encoding and decoding of android binary resources.
+//
+// Binary resource structs support unmarshalling the binary output of aapt.
+// Implementations of marshalling for each struct must produce the exact input
+// sent to unmarshalling. This allows tests to validate each struct representation
+// of the binary format as follows:
+//
+//  * unmarshal the output of aapt
+//  * marshal the struct representation
+//  * perform byte-to-byte comparison with aapt output per chunk header and body
+//
+// This process should strive to make structs idiomatic to make parsing xml text
+// into structs trivial.
+//
+// Once the struct representation is validated, tests for parsing xml text
+// into structs can become self-referential as the following holds true:
+//
+//  * the unmarshalled input of aapt output is the only valid target
+//  * the unmarshalled input of xml text may be compared to the unmarshalled
+//    input of aapt output to identify errors, e.g. text-trims, wrong flags, etc
+//
+// This provides validation, byte-for-byte, for producing binary xml resources.
+//
+// It should be made clear that unmarshalling binary resources is currently only
+// in scope for proving that the BinaryMarshaler works correctly. Any other use
+// is currently out of scope.
+//
+// A simple view of binary xml document structure:
+//
+//  XML
+//    Pool
+//    Map
+//    Namespace
+//    [...node]
+//
+// Additional resources:
+// https://android.googlesource.com/platform/frameworks/base/+/master/include/androidfw/ResourceTypes.h
+// https://justanapplication.wordpress.com/2011/09/13/ (a series of articles, increment date)
+package binres
+
+import (
+	"encoding"
+	"encoding/binary"
+	"fmt"
+)
+
+type ResType uint16
+
+func (t ResType) IsSupported() bool {
+	// explicit for clarity
+	return t == ResStringPool || t == ResXML ||
+		t == ResXMLStartNamespace || t == ResXMLEndNamespace ||
+		t == ResXMLStartElement || t == ResXMLEndElement ||
+		t == ResXMLCharData ||
+		t == ResXMLResourceMap ||
+		t == ResTable || t == ResTablePackage
+}
+
+// explicitly defined for clarity and resolvability with apt source
+const (
+	ResNull       ResType = 0x0000
+	ResStringPool ResType = 0x0001
+	ResTable      ResType = 0x0002
+	ResXML        ResType = 0x0003
+
+	ResXMLStartNamespace ResType = 0x0100
+	ResXMLEndNamespace   ResType = 0x0101
+	ResXMLStartElement   ResType = 0x0102
+	ResXMLEndElement     ResType = 0x0103
+	ResXMLCharData       ResType = 0x0104
+
+	ResXMLResourceMap ResType = 0x0180
+
+	ResTablePackage  ResType = 0x0200
+	ResTableType     ResType = 0x0201
+	ResTableTypeSpec ResType = 0x0202
+)
+
+var (
+	btou16 = binary.LittleEndian.Uint16
+	btou32 = binary.LittleEndian.Uint32
+	putu16 = binary.LittleEndian.PutUint16
+	putu32 = binary.LittleEndian.PutUint32
+)
+
+// unmarshaler wraps BinaryUnmarshaler to provide byte size of decoded chunks.
+type unmarshaler interface {
+	encoding.BinaryUnmarshaler
+
+	// size returns the byte size unmarshalled after a call to
+	// UnmarshalBinary, or otherwise zero.
+	size() int
+}
+
+// chunkHeader appears at the front of every data chunk in a resource.
+// TODO look into removing this, it's not necessary for marshalling and
+// the information provided may possibly be given more simply. For unmarshal,
+// a simple function would do.
+type chunkHeader struct {
+	// Type of data that follows this header.
+	typ ResType
+
+	// Advance slice index by this value to find its associated data, if any.
+	headerByteSize uint16
+
+	// This is the header size plus the size of any data associated with the chunk.
+	// Advance slice index by this value to completely skip its contents, including
+	// any child chunks. If this value is the same as headerByteSize, there is
+	// no data associated with the chunk.
+	byteSize uint32
+}
+
+// size implements unmarshaler.
+func (hdr chunkHeader) size() int { return int(hdr.byteSize) }
+
+func (hdr *chunkHeader) UnmarshalBinary(bin []byte) error {
+	hdr.typ = ResType(btou16(bin))
+	if !hdr.typ.IsSupported() {
+		return fmt.Errorf("%s not supported", hdr.typ)
+	}
+	hdr.headerByteSize = btou16(bin[2:])
+	hdr.byteSize = btou32(bin[4:])
+	return nil
+}
+
+func (hdr chunkHeader) MarshalBinary() ([]byte, error) {
+	if !hdr.typ.IsSupported() {
+		return nil, fmt.Errorf("%s not supported", hdr.typ)
+	}
+	bin := make([]byte, 8)
+	putu16(bin, uint16(hdr.typ))
+	putu16(bin[2:], hdr.headerByteSize)
+	putu32(bin[4:], hdr.byteSize)
+	return bin, nil
+}
+
+type XML struct {
+	chunkHeader
+
+	Pool *Pool
+	Map  *Map
+
+	Namespace *Namespace
+	Children  []*Element
+
+	// tmp field used when unmarshalling
+	stack []*Element
+}
+
+// TODO this is used strictly for querying in tests and dependent on
+// current XML.UnmarshalBinary implementation. Look into moving directly
+// into tests.
+var debugIndices = make(map[encoding.BinaryMarshaler]int)
+
+func (bx *XML) UnmarshalBinary(bin []byte) error {
+	buf := bin
+	if err := (&bx.chunkHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	buf = buf[8:]
+
+	// TODO this is tracked strictly for querying in tests; look into moving this
+	// functionality directly into tests if possible.
+	debugIndex := 8
+
+	for len(buf) > 0 {
+		t := ResType(btou16(buf))
+		k, err := bx.kind(t)
+		if err != nil {
+			return err
+		}
+		if err := k.UnmarshalBinary(buf); err != nil {
+			return err
+		}
+		debugIndices[k.(encoding.BinaryMarshaler)] = debugIndex
+		debugIndex += int(k.size())
+		buf = buf[k.size():]
+	}
+	return nil
+}
+
+func (bx *XML) kind(t ResType) (unmarshaler, error) {
+	switch t {
+	case ResStringPool:
+		if bx.Pool != nil {
+			return nil, fmt.Errorf("pool already exists")
+		}
+		bx.Pool = new(Pool)
+		return bx.Pool, nil
+	case ResXMLResourceMap:
+		if bx.Map != nil {
+			return nil, fmt.Errorf("resource map already exists")
+		}
+		bx.Map = new(Map)
+		return bx.Map, nil
+	case ResXMLStartNamespace:
+		if bx.Namespace != nil {
+			return nil, fmt.Errorf("namespace start already exists")
+		}
+		bx.Namespace = new(Namespace)
+		return bx.Namespace, nil
+	case ResXMLEndNamespace:
+		if bx.Namespace.end != nil {
+			return nil, fmt.Errorf("namespace end already exists")
+		}
+		bx.Namespace.end = new(Namespace)
+		return bx.Namespace.end, nil
+	case ResXMLStartElement:
+		el := new(Element)
+		if len(bx.stack) == 0 {
+			bx.Children = append(bx.Children, el)
+		} else {
+			n := len(bx.stack)
+			var p *Element
+			p, bx.stack = bx.stack[n-1], bx.stack[:n-1]
+			p.Children = append(p.Children, el)
+			bx.stack = append(bx.stack, p)
+		}
+		bx.stack = append(bx.stack, el)
+		return el, nil
+	case ResXMLEndElement:
+		n := len(bx.stack)
+		var el *Element
+		el, bx.stack = bx.stack[n-1], bx.stack[:n-1]
+		if el.end != nil {
+			return nil, fmt.Errorf("element end already exists")
+		}
+		el.end = new(ElementEnd)
+		return el.end, nil
+	case ResXMLCharData: // TODO
+		cdt := new(CharData)
+		el := bx.stack[len(bx.stack)-1]
+		if el.head == nil {
+			el.head = cdt
+		} else if el.tail == nil {
+			el.tail = cdt
+		} else {
+			return nil, fmt.Errorf("element head and tail already contain chardata")
+		}
+		return cdt, nil
+	default:
+		return nil, fmt.Errorf("unexpected type %s", t)
+	}
+}
+
+func (bx *XML) MarshalBinary() ([]byte, error) {
+	var (
+		bin, b []byte
+		err    error
+	)
+	b, err = bx.chunkHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	bin = append(bin, b...)
+
+	b, err = bx.Pool.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	bin = append(bin, b...)
+
+	b, err = bx.Map.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	bin = append(bin, b...)
+
+	b, err = bx.Namespace.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	bin = append(bin, b...)
+
+	for _, child := range bx.Children {
+		if err := marshalRecurse(child, &bin); err != nil {
+			return nil, err
+		}
+	}
+
+	b, err = bx.Namespace.end.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	bin = append(bin, b...)
+
+	return bin, nil
+}
+
+func marshalRecurse(el *Element, bin *[]byte) error {
+	b, err := el.MarshalBinary()
+	if err != nil {
+		return err
+	}
+	*bin = append(*bin, b...)
+
+	if el.head != nil {
+		b, err := el.head.MarshalBinary()
+		if err != nil {
+			return err
+		}
+		*bin = append(*bin, b...)
+	}
+
+	for _, child := range el.Children {
+		if err := marshalRecurse(child, bin); err != nil {
+			return err
+		}
+	}
+
+	b, err = el.end.MarshalBinary()
+	if err != nil {
+		return err
+	}
+	*bin = append(*bin, b...)
+
+	return nil
+}
+
+func (bx *XML) iterElements() <-chan *Element {
+	ch := make(chan *Element, 1)
+	go func() {
+		for _, el := range bx.Children {
+			iterElementsRecurse(el, ch)
+		}
+		close(ch)
+	}()
+	return ch
+}
+
+func iterElementsRecurse(el *Element, ch chan *Element) {
+	ch <- el
+	for _, e := range el.Children {
+		iterElementsRecurse(e, ch)
+	}
+}

internal/binres/binres_string.go

@@ -0,0 +1,87 @@
+// generated by stringer -output binres_string.go -type ResType,DataType; DO NOT EDIT
+
+package binres
+
+import "fmt"
+
+const (
+	_ResType_name_0 = "ResNullResStringPoolResTableResXML"
+	_ResType_name_1 = "ResXMLStartNamespaceResXMLEndNamespaceResXMLStartElementResXMLEndElementResXMLCharData"
+	_ResType_name_2 = "ResXMLResourceMap"
+	_ResType_name_3 = "ResTablePackageResTableTypeResTableTypeSpec"
+)
+
+var (
+	_ResType_index_0 = [...]uint8{7, 20, 28, 34}
+	_ResType_index_1 = [...]uint8{20, 38, 56, 72, 86}
+	_ResType_index_2 = [...]uint8{17}
+	_ResType_index_3 = [...]uint8{15, 27, 43}
+)
+
+func (i ResType) String() string {
+	switch {
+	case 0 <= i && i <= 3:
+		lo := uint8(0)
+		if i > 0 {
+			lo = _ResType_index_0[i-1]
+		}
+		return _ResType_name_0[lo:_ResType_index_0[i]]
+	case 256 <= i && i <= 260:
+		i -= 256
+		lo := uint8(0)
+		if i > 0 {
+			lo = _ResType_index_1[i-1]
+		}
+		return _ResType_name_1[lo:_ResType_index_1[i]]
+	case i == 384:
+		return _ResType_name_2
+	case 512 <= i && i <= 514:
+		i -= 512
+		lo := uint8(0)
+		if i > 0 {
+			lo = _ResType_index_3[i-1]
+		}
+		return _ResType_name_3[lo:_ResType_index_3[i]]
+	default:
+		return fmt.Sprintf("ResType(%d)", i)
+	}
+}
+
+const (
+	_DataType_name_0 = "DataNullDataReferenceDataAttributeDataStringDataFloatDataDimensionDataFractionDataDynamicReference"
+	_DataType_name_1 = "DataIntDecDataIntHexDataIntBool"
+	_DataType_name_2 = "DataIntColorARGB8DataIntColorRGB8DataIntColorARGB4DataIntColorRGB4"
+)
+
+var (
+	_DataType_index_0 = [...]uint8{8, 21, 34, 44, 53, 66, 78, 98}
+	_DataType_index_1 = [...]uint8{10, 20, 31}
+	_DataType_index_2 = [...]uint8{17, 33, 50, 66}
+)
+
+func (i DataType) String() string {
+	switch {
+	case 0 <= i && i <= 7:
+		lo := uint8(0)
+		if i > 0 {
+			lo = _DataType_index_0[i-1]
+		}
+		return _DataType_name_0[lo:_DataType_index_0[i]]
+	case 16 <= i && i <= 18:
+		i -= 16
+		lo := uint8(0)
+		if i > 0 {
+			lo = _DataType_index_1[i-1]
+		}
+		return _DataType_name_1[lo:_DataType_index_1[i]]
+	case 28 <= i && i <= 31:
+		i -= 28
+		lo := uint8(0)
+		if i > 0 {
+			lo = _DataType_index_2[i-1]
+		}
+		return _DataType_name_2[lo:_DataType_index_2[i]]
+	default:
+		return fmt.Sprintf("DataType(%d)", i)
+	}
+}

internal/binres/binres_test.go

@@ -0,0 +1,271 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package binres
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func printrecurse(t *testing.T, pl *Pool, el *Element, ws string) {
+	for _, attr := range el.attrs {
+		ns := ""
+		if attr.NS != math.MaxUint32 {
+			ns = pl.strings[int(attr.NS)]
+			nss := strings.Split(ns, "/")
+			ns = nss[len(nss)-1]
+		}
+
+		val := ""
+		if attr.RawValue != math.MaxUint32 {
+			val = pl.strings[int(attr.RawValue)]
+		} else {
+			switch attr.TypedValue.Type {
+			case DataIntDec:
+				val = fmt.Sprintf("%v", attr.TypedValue.Value)
+			case DataIntBool:
+				val = fmt.Sprintf("%v", attr.TypedValue.Value == 1)
+			default:
+				val = fmt.Sprintf("0x%08X", attr.TypedValue.Value)
+			}
+		}
+		dt := attr.TypedValue.Type
+
+		t.Logf("%s|attr:ns(%v) name(%s) val(%s) valtyp(%s)\n", ws, ns, pl.strings[int(attr.Name)], val, dt)
+	}
+	t.Log()
+	for _, e := range el.Children {
+		printrecurse(t, pl, e, ws+"        ")
+	}
+}
+
+func TestBootstrap(t *testing.T) {
+	bin, err := ioutil.ReadFile("testdata/bootstrap.bin")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	checkMarshal := func(res encoding.BinaryMarshaler, bsize int) {
+		b, err := res.MarshalBinary()
+		if err != nil {
+			t.Error(err)
+		}
+		idx := debugIndices[res]
+		a := bin[idx : idx+bsize]
+		if !bytes.Equal(a, b) {
+			x, y := len(a), len(b)
+			if x != y {
+				t.Errorf("%v: %T: byte length does not match, have %v, want %v", idx, res, y, x)
+			}
+			if x > y {
+				x, y = y, x
+			}
+			mismatch := false
+			for i := 0; i < x; i++ {
+				if mismatch = a[i] != b[i]; mismatch {
+					t.Errorf("%v: %T: first byte mismatch at %v of %v", idx, res, i, bsize)
+					break
+				}
+			}
+			if mismatch {
+				// print out a reasonable amount of data to help identify issues
+				truncate := x > 1300
+				if truncate {
+					x = 1300
+				}
+				t.Log("      HAVE               WANT")
+				for i := 0; i < x; i += 4 {
+					he, we := 4, 4
+					if i+he >= x {
+						he = x - i
+					}
+					if i+we >= y {
+						we = y - i
+					}
+					t.Logf("%3v | % X        % X\n", i, b[i:i+he], a[i:i+we])
+				}
+				if truncate {
+					t.Log("... output truncated.")
+				}
+			}
+		}
+	}
+
+	bxml := new(XML)
+	if err := bxml.UnmarshalBinary(bin); err != nil {
+		t.Fatal(err)
+	}
+
+	for i, x := range bxml.Pool.strings {
+		t.Logf("Pool(%v): %q\n", i, x)
+	}
+
+	for _, e := range bxml.Children {
+		printrecurse(t, bxml.Pool, e, "")
+	}
+
+	checkMarshal(&bxml.chunkHeader, int(bxml.headerByteSize))
+	checkMarshal(bxml.Pool, bxml.Pool.size())
+	checkMarshal(bxml.Map, bxml.Map.size())
+	checkMarshal(bxml.Namespace, bxml.Namespace.size())
+
+	for el := range bxml.iterElements() {
+		checkMarshal(el, el.size())
+		checkMarshal(el.end, el.end.size())
+	}
+
+	checkMarshal(bxml.Namespace.end, bxml.Namespace.end.size())
+	checkMarshal(bxml, bxml.size())
+}
+
+func retset(xs []string) []string {
+	m := make(map[string]struct{})
+	fo := xs[:0]
+	for _, x := range xs {
+		if x == "" {
+			continue
+		}
+		if _, ok := m[x]; !ok {
+			m[x] = struct{}{}
+			fo = append(fo, x)
+		}
+	}
+	return fo
+}
+
+type ByNamespace []xml.Attr
+
+func (a ByNamespace) Len() int      { return len(a) }
+func (a ByNamespace) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByNamespace) Less(i, j int) bool {
+	if a[j].Name.Space == "" {
+		return a[i].Name.Space != ""
+	}
+	return false
+}
+
+// WIP approximation of first steps to be taken to encode manifest
+func TestEncode(t *testing.T) {
+	f, err := os.Open("testdata/bootstrap.xml")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var attrs []xml.Attr
+
+	dec := xml.NewDecoder(f)
+	for {
+		tkn, err := dec.Token()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return
+			// t.Fatal(err)
+		}
+		tkn = xml.CopyToken(tkn)
+
+		switch tkn := tkn.(type) {
+		case xml.StartElement:
+			attrs = append(attrs, tkn.Attr...)
+		default:
+			// t.Error("unhandled token type", tkn)
+		}
+	}
+
+	bvc := xml.Attr{
+		Name: xml.Name{
+			Space: "",
+			Local: "platformBuildVersionCode",
+		},
+		Value: "10",
+	}
+	bvn := xml.Attr{
+		Name: xml.Name{
+			Space: "",
+			Local: "platformBuildVersionName",
+		},
+		Value: "2.3.3",
+	}
+	attrs = append(attrs, bvc, bvn)
+
+	sort.Sort(ByNamespace(attrs))
+	var names, vals []string
+	for _, attr := range attrs {
+		if strings.HasSuffix(attr.Name.Space, "tools") {
+			continue
+		}
+		names = append(names, attr.Name.Local)
+		vals = append(vals, attr.Value)
+	}
+
+	var all []string
+	all = append(all, names...)
+	all = append(all, vals...)
+
+	// do not eliminate duplicates until the entire slice has been composed.
+	// consider <activity android:label="label" .../>
+	// all attribute names come first followed by values; in such a case, the value "label"
+	// would be a reference to the same "android:label" in the string pool which will occur
+	// within the beginning of the pool where other attr names are located.
+	pl := new(Pool)
+	for _, x := range retset(all) {
+		pl.strings = append(pl.strings, x)
+		// t.Logf("Pool(%v) %q\n", i, x)
+	}
+}
+
+func TestAndroidJar(t *testing.T) {
+	zr, err := zip.OpenReader("/home/daniel/local/android-sdk/platforms/android-10/android.jar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer zr.Close()
+
+	buf := new(bytes.Buffer)
+	for _, f := range zr.File {
+		if f.Name == "resources.arsc" {
+			rc, err := f.Open()
+			if err != nil {
+				t.Fatal(err)
+			}
+			_, err = io.Copy(buf, rc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rc.Close()
+			break
+		}
+	}
+	if buf.Len() == 0 {
+		t.Fatal("failed to read resources.arsc")
+	}
+
+	bin := buf.Bytes()
+
+	tbl := &Table{}
+	if err := tbl.UnmarshalBinary(bin); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("%+v\n", tbl.TableHeader)
+	t.Logf("%+v\n", tbl.pool.chunkHeader)
+	for i, x := range tbl.pool.strings[:10] {
+		t.Logf("pool(%v) %s\n", i, x)
+	}
+
+	t.Logf("%+v\n", tbl.pkg)
+}

internal/binres/data.go

@@ -0,0 +1,50 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package binres
+
+type DataType uint8
+
+// explicitly defined for clarity and resolvability with apt source
+const (
+	DataNull             DataType = 0x00 // either 0 or 1 for resource undefined or empty
+	DataReference        DataType = 0x01 // ResTable_ref, a reference to another resource table entry
+	DataAttribute        DataType = 0x02 // attribute resource identifier
+	DataString           DataType = 0x03 // index into the containing resource table's global value string pool
+	DataFloat            DataType = 0x04 // single-precision floating point number
+	DataDimension        DataType = 0x05 // complex number encoding a dimension value, such as "100in"
+	DataFraction         DataType = 0x06 // complex number encoding a fraction of a container
+	DataDynamicReference DataType = 0x07 // dynamic ResTable_ref, which needs to be resolved before it can be used like a TYPE_REFERENCE.
+	DataIntDec           DataType = 0x10 // raw integer value of the form n..n
+	DataIntHex           DataType = 0x11 // raw integer value of the form 0xn..n
+	DataIntBool          DataType = 0x12 // either 0 or 1, for input "false" or "true"
+	DataIntColorARGB8    DataType = 0x1c // raw integer value of the form #aarrggbb
+	DataIntColorRGB8     DataType = 0x1d // raw integer value of the form #rrggbb
+	DataIntColorARGB4    DataType = 0x1e // raw integer value of the form #argb
+	DataIntColorRGB4     DataType = 0x1f // raw integer value of the form #rgb
+)
+
+type Data struct {
+	ByteSize uint16
+	Res0     uint8 // always 0, useful for debugging bad read offsets
+	Type     DataType
+	Value    uint32
+}
+
+func (d *Data) UnmarshalBinary(bin []byte) error {
+	d.ByteSize = btou16(bin)
+	d.Res0 = uint8(bin[2])
+	d.Type = DataType(bin[3])
+	d.Value = btou32(bin[4:])
+	return nil
+}
+
+func (d *Data) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 8)
+	putu16(bin, d.ByteSize)
+	bin[2] = byte(d.Res0)
+	bin[3] = byte(d.Type)
+	putu32(bin[4:], d.Value)
+	return bin, nil
+}

internal/binres/node.go

@@ -0,0 +1,225 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package binres
+
+// NodeHeader is header all xml node types have, providing additional
+// information regarding an xml node over binChunkHeader.
+type NodeHeader struct {
+	chunkHeader
+	LineNumber uint32  // line number in source file this element appears
+	Comment    PoolRef // optional xml comment associated with element, MaxUint32 if none
+}
+
+func (hdr *NodeHeader) UnmarshalBinary(bin []byte) error {
+	if err := (&hdr.chunkHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	hdr.LineNumber = btou32(bin[8:])
+	hdr.Comment = PoolRef(btou32(bin[12:]))
+	return nil
+}
+
+func (hdr *NodeHeader) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 16)
+	b, err := hdr.chunkHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	putu32(bin[8:], hdr.LineNumber)
+	putu32(bin[12:], uint32(hdr.Comment))
+	return bin, nil
+}
+
+type Namespace struct {
+	NodeHeader
+	prefix PoolRef
+	uri    PoolRef
+
+	end *Namespace // TODO don't let this type be recursive
+}
+
+func (ns *Namespace) UnmarshalBinary(bin []byte) error {
+	if err := (&ns.NodeHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	buf := bin[ns.headerByteSize:]
+	ns.prefix = PoolRef(btou32(buf))
+	ns.uri = PoolRef(btou32(buf[4:]))
+	return nil
+}
+
+func (ns *Namespace) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 24)
+	b, err := ns.NodeHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	putu32(bin[16:], uint32(ns.prefix))
+	putu32(bin[20:], uint32(ns.uri))
+	return bin, nil
+}
+
+type Element struct {
+	NodeHeader
+	NS             PoolRef
+	Name           PoolRef // name of node if element, otherwise chardata if CDATA
+	AttributeStart uint16  // byte offset where attrs start
+	AttributeSize  uint16  // byte size of attrs
+	AttributeCount uint16  // length of attrs
+	IdIndex        uint16  // Index (1-based) of the "id" attribute. 0 if none.
+	ClassIndex     uint16  // Index (1-based) of the "class" attribute. 0 if none.
+	StyleIndex     uint16  // Index (1-based) of the "style" attribute. 0 if none.
+
+	attrs    []*Attribute
+	Children []*Element
+	end      *ElementEnd
+
+	head, tail *CharData
+}
+
+func (el *Element) UnmarshalBinary(bin []byte) error {
+	if err := (&el.NodeHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	buf := bin[el.headerByteSize:] // 16
+	el.NS = PoolRef(btou32(buf))
+	el.Name = PoolRef(btou32(buf[4:]))
+	el.AttributeStart = btou16(buf[8:])
+	el.AttributeSize = btou16(buf[10:])
+	el.AttributeCount = btou16(buf[12:])
+	el.IdIndex = btou16(buf[14:])
+	el.ClassIndex = btou16(buf[16:])
+	el.StyleIndex = btou16(buf[18:])
+
+	buf = buf[el.AttributeStart:]
+	el.attrs = make([]*Attribute, int(el.AttributeCount))
+	for i := range el.attrs {
+		attr := new(Attribute)
+		if err := attr.UnmarshalBinary(buf); err != nil {
+			return err
+		}
+		el.attrs[i] = attr
+		buf = buf[el.AttributeSize:]
+	}
+
+	return nil
+}
+
+func (el *Element) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 16+20+len(el.attrs)*int(el.AttributeSize))
+	b, err := el.NodeHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	putu32(bin[16:], uint32(el.NS))
+	putu32(bin[20:], uint32(el.Name))
+	putu16(bin[24:], el.AttributeStart)
+	putu16(bin[26:], el.AttributeSize)
+	putu16(bin[28:], el.AttributeCount)
+	putu16(bin[30:], el.IdIndex)
+	putu16(bin[32:], el.ClassIndex)
+	putu16(bin[34:], el.StyleIndex)
+
+	buf := bin[36:]
+	for _, attr := range el.attrs {
+		b, err := attr.MarshalBinary()
+		if err != nil {
+			return nil, err
+		}
+		copy(buf, b)
+		buf = buf[int(el.AttributeSize):]
+	}
+
+	return bin, nil
+}
+
+// ElementEnd marks the end of an element node, either Element or CharData.
+type ElementEnd struct {
+	NodeHeader
+	NS   PoolRef
+	Name PoolRef // name of node if binElement, raw chardata if binCharData
+}
+
+func (el *ElementEnd) UnmarshalBinary(bin []byte) error {
+	(&el.NodeHeader).UnmarshalBinary(bin)
+	buf := bin[el.headerByteSize:]
+	el.NS = PoolRef(btou32(buf))
+	el.Name = PoolRef(btou32(buf[4:]))
+	return nil
+}
+
+func (el *ElementEnd) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 24)
+	b, err := el.NodeHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	putu32(bin[16:], uint32(el.NS))
+	putu32(bin[20:], uint32(el.Name))
+	return bin, nil
+}
+
+type Attribute struct {
+	NS         PoolRef
+	Name       PoolRef
+	RawValue   PoolRef // The original raw string value of this attribute.
+	TypedValue Data    // Processesd typed value of this attribute.
+}
+
+func (attr *Attribute) UnmarshalBinary(bin []byte) error {
+	attr.NS = PoolRef(btou32(bin))
+	attr.Name = PoolRef(btou32(bin[4:]))
+	attr.RawValue = PoolRef(btou32(bin[8:]))
+	return (&attr.TypedValue).UnmarshalBinary(bin[12:])
+}
+
+func (attr *Attribute) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 20)
+	putu32(bin, uint32(attr.NS))
+	putu32(bin[4:], uint32(attr.Name))
+	putu32(bin[8:], uint32(attr.RawValue))
+	b, err := attr.TypedValue.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin[12:], b)
+	return bin, nil
+}
+
+// CharData represents a CDATA node and includes ref to node's text value.
+type CharData struct {
+	NodeHeader
+	RawData   PoolRef // raw character data
+	TypedData Data    // typed value of character data
+}
+
+func (cdt *CharData) UnmarshalBinary(bin []byte) error {
+	if err := (&cdt.NodeHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	buf := bin[cdt.headerByteSize:]
+	cdt.RawData = PoolRef(btou32(buf))
+	return (&cdt.TypedData).UnmarshalBinary(buf[4:])
+}
+
+func (cdt *CharData) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 28)
+	b, err := cdt.NodeHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	putu32(bin[16:], uint32(cdt.RawData))
+	b, err = cdt.TypedData.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin[20:], b)
+	return bin, nil
+}

internal/binres/package.go

@@ -0,0 +1,101 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package binres
+
+type TableHeader struct {
+	chunkHeader
+	packageCount uint32
+}
+
+func (hdr *TableHeader) UnmarshalBinary(bin []byte) error {
+	if err := (&hdr.chunkHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	hdr.packageCount = btou32(bin[8:])
+	return nil
+}
+
+func (hdr TableHeader) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 12)
+	b, err := hdr.chunkHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	putu32(b[8:], hdr.packageCount)
+	return bin, nil
+}
+
+// TODO next up: package chunk
+// https://justanapplication.wordpress.com/2011/09/16/
+type Table struct {
+	TableHeader
+	pool *Pool
+	pkg  *Package
+}
+
+func (tbl *Table) UnmarshalBinary(bin []byte) error {
+	buf := bin
+	if err := (&tbl.TableHeader).UnmarshalBinary(buf); err != nil {
+		return err
+	}
+	buf = buf[tbl.headerByteSize:]
+	tbl.pool = new(Pool)
+	if err := tbl.pool.UnmarshalBinary(buf); err != nil {
+		return err
+	}
+	buf = buf[tbl.pool.size():]
+	tbl.pkg = new(Package)
+	if err := tbl.pkg.UnmarshalBinary(buf); err != nil {
+		return err
+	}
+	return nil
+}
+
+type Package struct {
+	chunkHeader
+
+	// If this is a base package, its ID.  Package IDs start
+	// at 1 (corresponding to the value of the package bits in a
+	// resource identifier).  0 means this is not a base package.
+	id uint32
+
+	// name of package, zero terminated
+	name [128]uint16
+
+	// Offset to a ResStringPool_header defining the resource
+	// type symbol table.  If zero, this package is inheriting from
+	// another base package (overriding specific values in it).
+	typeStrings uint32
+
+	// Last index into typeStrings that is for public use by others.
+	lastPublicType uint32
+
+	// Offset to a ResStringPool_header defining the resource
+	// key symbol table.  If zero, this package is inheriting from
+	// another base package (overriding specific values in it).
+	keyStrings uint32
+
+	// Last index into keyStrings that is for public use by others.
+	lastPublicKey uint32
+
+	typeIdOffset uint32
+}
+
+func (pkg *Package) UnmarshalBinary(bin []byte) error {
+	if err := (&pkg.chunkHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	pkg.id = btou32(bin[8:])
+	for i := range pkg.name {
+		pkg.name[i] = btou16(bin[12+i*2:])
+	}
+	pkg.typeStrings = btou32(bin[140:])
+	pkg.lastPublicType = btou32(bin[144:])
+	pkg.keyStrings = btou32(bin[148:])
+	pkg.lastPublicKey = btou32(bin[152:])
+	pkg.typeIdOffset = btou32(bin[156:])
+	return nil
+}

internal/binres/pool.go

@@ -0,0 +1,276 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package binres
+
+import (
+	"fmt"
+	"unicode/utf16"
+)
+
+const (
+	SortedFlag uint32 = 1 << 0
+	UTF8Flag          = 1 << 8
+)
+
+// PoolRef is the i'th string in a pool.
+type PoolRef uint32
+
+// Pool has the following structure marshalled:
+//
+// 	binChunkHeader
+//  StringCount  uint32 // Number of strings in this pool
+//  StyleCount   uint32 // Number of style spans in pool
+//  Flags        uint32 // SortedFlag, UTF8Flag
+//  StringsStart uint32 // Index of string data from header
+//  StylesStart  uint32 // Index of style data from header
+//
+//  StringIndices []uint32 // starting at zero
+//
+//  // UTF16 entries are concatenations of the following:
+//  // [2]byte uint16 string length, exclusive
+//  // [2]byte [optional] low word if high bit of length was set
+//  // [n]byte data
+//  // [2]byte 0x0000 terminator
+//  Strings []uint16
+type Pool struct {
+	chunkHeader
+
+	strings []string
+	styles  []*Span
+	flags   uint32 // SortedFlag, UTF8Flag
+}
+
+func (pl *Pool) IsSorted() bool { return pl.flags&SortedFlag == SortedFlag }
+func (pl *Pool) IsUTF8() bool   { return pl.flags&UTF8Flag == UTF8Flag }
+
+func (pl *Pool) UnmarshalBinary(bin []byte) error {
+	if err := (&pl.chunkHeader).UnmarshalBinary(bin); err != nil {
+		return err
+	}
+	if pl.typ != ResStringPool {
+		return fmt.Errorf("have type %s, want %s", pl.typ, ResStringPool)
+	}
+
+	nstrings := btou32(bin[8:])
+	pl.strings = make([]string, nstrings)
+
+	nstyles := btou32(bin[12:])
+	pl.styles = make([]*Span, nstyles)
+
+	pl.flags = btou32(bin[16:])
+
+	offstrings := btou32(bin[20:])
+	offstyle := btou32(bin[24:])
+
+	hdrlen := 28 // header size is always 28
+	if pl.IsUTF8() {
+		for i := range pl.strings {
+			ii := btou32(bin[hdrlen+i*4:]) // index of string index
+			r := int(offstrings + ii)      // read index of string
+
+			// for char and byte sizes below, if leading bit set,
+			// treat first as 7-bit high word and the next byte as low word.
+
+			// get char size of string
+			cn := uint8(bin[r])
+			rcn := int(cn)
+			r++
+			if cn&(1<<7) != 0 {
+				cn0 := int(cn ^ (1 << 7)) // high word
+				cn1 := int(bin[r])        // low word
+				rcn = cn0*256 + cn1
+				r++
+			}
+
+			// get byte size of string
+			// TODO(d) i've seen at least one case in android.jar resource table that has only
+			// highbit set, effectively making 7-bit highword zero. The reason for this is currently
+			// unknown but would make tests that unmarshal-marshal to match bytes impossible to pass.
+			// The values noted were high-word: 0 (after highbit unset), low-word: 141
+			// I don't recall character count but was around ?78?
+			// The case here may very well be that the size is treated like int8 triggering the use of
+			// two bytes to store size even though the implementation uses uint8.
+			n := uint8(bin[r])
+			r++
+			rn := int(n)
+			if n&(1<<7) != 0 {
+				n0 := int(n ^ (1 << 7)) // high word
+				n1 := int(bin[r])       // low word
+				rn = n0*(1<<8) + n1
+				r++
+			}
+
+			//
+			data := bin[r : r+rn]
+			if x := uint8(bin[r+rn]); x != 0 {
+				return fmt.Errorf("expected zero terminator, got %v for byte size %v char len %v", x, rn, rcn)
+			}
+			pl.strings[i] = string(data)
+		}
+	} else {
+		for i := range pl.strings {
+			ii := btou32(bin[hdrlen+i*4:]) // index of string index
+			r := int(offstrings + ii)      // read index of string
+			n := btou16(bin[r:])           // string length
+			rn := int(n)
+			r += 2
+
+			if n&(1<<15) != 0 { // TODO this is untested
+				n0 := int(n ^ (1 << 15)) // high word
+				n1 := int(btou16(bin[r:]))
+				rn = n0*(1<<16) + n1
+				r += 2
+			}
+
+			data := make([]uint16, int(rn))
+			for i := range data {
+				data[i] = btou16(bin[r+(2*i):])
+			}
+
+			r += int(n * 2)
+			if x := btou16(bin[r:]); x != 0 {
+				return fmt.Errorf("expected zero terminator, got 0x%04X\n%s", x)
+			}
+
+			pl.strings[i] = string(utf16.Decode(data))
+		}
+	}
+
+	// TODO
+	_ = offstyle
+	// styii := hdrlen + int(nstrings*4)
+	// for i := range pl.styles {
+	// ii := btou32(bin[styii+i*4:])
+	// r := int(offstyle + ii)
+	// spn := new(binSpan)
+	// spn.UnmarshalBinary(bin[r:])
+	// pl.styles[i] = spn
+	// }
+
+	return nil
+}
+
+func (pl *Pool) MarshalBinary() ([]byte, error) {
+	if pl.IsUTF8() {
+		return nil, fmt.Errorf("encode utf8 not supported")
+	}
+
+	var (
+		hdrlen = 28
+		// indices of string indices
+		iis    = make([]uint32, len(pl.strings))
+		iislen = len(iis) * 4
+		// utf16 encoded strings concatenated together
+		strs []uint16
+	)
+	for i, x := range pl.strings {
+		if len(x)>>16 > 0 {
+			panic(fmt.Errorf("string lengths over 1<<15 not yet supported, got len %d", len(x)))
+		}
+		p := utf16.Encode([]rune(x))
+		if len(p) == 0 {
+			strs = append(strs, 0x0000, 0x0000)
+		} else {
+			strs = append(strs, uint16(len(p))) // string length (implicitly includes zero terminator to follow)
+			strs = append(strs, p...)
+			strs = append(strs, 0) // zero terminated
+		}
+		// indices start at zero
+		if i+1 != len(iis) {
+			iis[i+1] = uint32(len(strs) * 2) // utf16 byte index
+		}
+	}
+
+	// check strings is 4-byte aligned, pad with zeros if not.
+	for x := (len(strs) * 2) % 4; x != 0; x -= 2 {
+		strs = append(strs, 0x0000)
+	}
+
+	strslen := len(strs) * 2
+
+	hdr := chunkHeader{
+		typ:            ResStringPool,
+		headerByteSize: 28,
+		byteSize:       uint32(28 + iislen + strslen),
+	}
+
+	bin := make([]byte, hdr.byteSize)
+
+	hdrbin, err := hdr.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, hdrbin)
+
+	putu32(bin[8:], uint32(len(pl.strings)))
+	putu32(bin[12:], uint32(len(pl.styles)))
+	putu32(bin[16:], pl.flags)
+	putu32(bin[20:], uint32(hdrlen+iislen))
+	putu32(bin[24:], 0) // index of styles start, is 0 when styles length is 0
+
+	buf := bin[28:]
+	for _, x := range iis {
+		putu32(buf, x)
+		buf = buf[4:]
+	}
+	for _, x := range strs {
+		putu16(buf, x)
+		buf = buf[2:]
+	}
+
+	if len(buf) != 0 {
+		panic(fmt.Errorf("failed to fill allocated buffer, %v bytes left over", len(buf)))
+	}
+
+	return bin, nil
+}
+
+type Span struct {
+	name PoolRef
+
+	firstChar, lastChar uint32
+}
+
+func (spn *Span) UnmarshalBinary(bin []byte) error {
+	const end = 0xFFFFFFFF
+	spn.name = PoolRef(btou32(bin))
+	if spn.name == end {
+		return nil
+	}
+	spn.firstChar = btou32(bin[4:])
+	spn.lastChar = btou32(bin[8:])
+	return nil
+}
+
+// Map contains a uint32 slice mapping strings in the string
+// pool back to resource identifiers. The i'th element of the slice
+// is also the same i'th element of the string pool.
+type Map struct {
+	chunkHeader
+	rs []uint32
+}
+
+func (m *Map) UnmarshalBinary(bin []byte) error {
+	(&m.chunkHeader).UnmarshalBinary(bin)
+	buf := bin[m.headerByteSize:m.byteSize]
+	m.rs = make([]uint32, len(buf)/4)
+	for i := range m.rs {
+		m.rs[i] = btou32(buf[i*4:])
+	}
+	return nil
+}
+
+func (m *Map) MarshalBinary() ([]byte, error) {
+	bin := make([]byte, 8+len(m.rs)*4)
+	b, err := m.chunkHeader.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	copy(bin, b)
+	for i, r := range m.rs {
+		putu32(bin[8+i*4:], r)
+	}
+	return bin, nil
+}

internal/binres/testdata/bootstrap.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+Copyright 2014 The Go Authors. All rights reserved.
+Use of this source code is governed by a BSD-style
+license that can be found in the LICENSE file.
+-->
+<manifest
+	xmlns:android="http://schemas.android.com/apk/res/android"
+	package="com.zentus.balloon"
+	android:versionCode="42"
+	android:versionName=""
+	xmlns:tools="http://schemas.android.com/tools">
+
+	<uses-sdk android:minSdkVersion="9" />
+	<application
+			android:label="Balloon世界"
+			android:hasCode="false"
+			android:debuggable="true"
+			tools:strict="label">
+	<activity android:name="android.app.NativeActivity"
+		android:theme="@android:style/Theme.NoTitleBar.Fullscreen"
+		android:label="Balloon"
+		android:screenOrientation="portrait"
+		android:configChanges="orientation|keyboardHidden">
+		<meta-data android:name="android.app.lib_name" android:value="balloon" />
+		<intent-filter>
+			here is some text
+			<action android:name="android.intent.action.MAIN" />
+			<category android:name="android.intent.category.LAUNCHER" />
+		</intent-filter>
+	</activity>
+	</application>
+</manifest>

internal/binres/testdata/gen.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/sh
+
+# version of build-tools tests run against
+AAPT=${ANDROID_HOME}/build-tools/23.0.1/aapt
+
+# minimum version of android api for resource identifiers supported
+APIJAR=${ANDROID_HOME}/platforms/android-10/android.jar
+
+for f in *.xml; do
+	cp "$f" AndroidManifest.xml
+	"$AAPT" p -M AndroidManifest.xml -I "$APIJAR" -F tmp.apk
+	unzip -qq -o tmp.apk
+	mv AndroidManifest.xml "${f:0:-3}bin"
+	rm tmp.apk
+done