From c897050410ed0994fca72ce1151681b435cc8950 Mon Sep 17 00:00:00 2001 From: Daniel Skinner Date: Mon, 26 Oct 2015 15:55:44 -0500 Subject: [PATCH] cmd/gomobile: new binary resource implementation Change-Id: I9708170ac2c5914bb8e978b4703f4e6f7415c737 Reviewed-on: https://go-review.googlesource.com/16553 Reviewed-by: David Crawshaw --- internal/binres/binres.go | 342 +++++++++++++++++++++++++ internal/binres/binres_string.go | 87 +++++++ internal/binres/binres_test.go | 271 ++++++++++++++++++++ internal/binres/data.go | 50 ++++ internal/binres/node.go | 225 ++++++++++++++++ internal/binres/package.go | 101 ++++++++ internal/binres/pool.go | 276 ++++++++++++++++++++ internal/binres/testdata/bootstrap.bin | Bin 0 -> 2188 bytes internal/binres/testdata/bootstrap.xml | 33 +++ internal/binres/testdata/gen.sh | 15 ++ 10 files changed, 1400 insertions(+) create mode 100644 internal/binres/binres.go create mode 100644 internal/binres/binres_string.go create mode 100644 internal/binres/binres_test.go create mode 100644 internal/binres/data.go create mode 100644 internal/binres/node.go create mode 100644 internal/binres/package.go create mode 100644 internal/binres/pool.go create mode 100644 internal/binres/testdata/bootstrap.bin create mode 100644 internal/binres/testdata/bootstrap.xml create mode 100755 internal/binres/testdata/gen.sh diff --git a/internal/binres/binres.go b/internal/binres/binres.go new file mode 100644 index 0000000..d9e792f --- /dev/null +++ b/internal/binres/binres.go @@ -0,0 +1,342 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate stringer -output binres_string.go -type ResType,DataType + +// Package binres implements encoding and decoding of android binary resources. +// +// Binary resource structs support unmarshalling the binary output of aapt. +// Implementations of marshalling for each struct must produce the exact input +// sent to unmarshalling. This allows tests to validate each struct representation +// of the binary format as follows: +// +// * unmarshal the output of aapt +// * marshal the struct representation +// * perform byte-to-byte comparison with aapt output per chunk header and body +// +// This process should strive to make structs idiomatic to make parsing xml text +// into structs trivial. +// +// Once the struct representation is validated, tests for parsing xml text +// into structs can become self-referential as the following holds true: +// +// * the unmarshalled input of aapt output is the only valid target +// * the unmarshalled input of xml text may be compared to the unmarshalled +// input of aapt output to identify errors, e.g. text-trims, wrong flags, etc +// +// This provides validation, byte-for-byte, for producing binary xml resources. +// +// It should be made clear that unmarshalling binary resources is currently only +// in scope for proving that the BinaryMarshaler works correctly. Any other use +// is currently out of scope. +// +// A simple view of binary xml document structure: +// +// XML +// Pool +// Map +// Namespace +// [...node] +// +// Additional resources: +// https://android.googlesource.com/platform/frameworks/base/+/master/include/androidfw/ResourceTypes.h +// https://justanapplication.wordpress.com/2011/09/13/ (a series of articles, increment date) +package binres + +import ( + "encoding" + "encoding/binary" + "fmt" +) + +type ResType uint16 + +func (t ResType) IsSupported() bool { + // explicit for clarity + return t == ResStringPool || t == ResXML || + t == ResXMLStartNamespace || t == ResXMLEndNamespace || + t == ResXMLStartElement || t == ResXMLEndElement || + t == ResXMLCharData || + t == ResXMLResourceMap || + t == ResTable || t == ResTablePackage +} + +// explicitly defined for clarity and resolvability with apt source +const ( + ResNull ResType = 0x0000 + ResStringPool ResType = 0x0001 + ResTable ResType = 0x0002 + ResXML ResType = 0x0003 + + ResXMLStartNamespace ResType = 0x0100 + ResXMLEndNamespace ResType = 0x0101 + ResXMLStartElement ResType = 0x0102 + ResXMLEndElement ResType = 0x0103 + ResXMLCharData ResType = 0x0104 + + ResXMLResourceMap ResType = 0x0180 + + ResTablePackage ResType = 0x0200 + ResTableType ResType = 0x0201 + ResTableTypeSpec ResType = 0x0202 +) + +var ( + btou16 = binary.LittleEndian.Uint16 + btou32 = binary.LittleEndian.Uint32 + putu16 = binary.LittleEndian.PutUint16 + putu32 = binary.LittleEndian.PutUint32 +) + +// unmarshaler wraps BinaryUnmarshaler to provide byte size of decoded chunks. +type unmarshaler interface { + encoding.BinaryUnmarshaler + + // size returns the byte size unmarshalled after a call to + // UnmarshalBinary, or otherwise zero. + size() int +} + +// chunkHeader appears at the front of every data chunk in a resource. +// TODO look into removing this, it's not necessary for marshalling and +// the information provided may possibly be given more simply. For unmarshal, +// a simple function would do. +type chunkHeader struct { + // Type of data that follows this header. + typ ResType + + // Advance slice index by this value to find its associated data, if any. + headerByteSize uint16 + + // This is the header size plus the size of any data associated with the chunk. + // Advance slice index by this value to completely skip its contents, including + // any child chunks. If this value is the same as headerByteSize, there is + // no data associated with the chunk. + byteSize uint32 +} + +// size implements unmarshaler. +func (hdr chunkHeader) size() int { return int(hdr.byteSize) } + +func (hdr *chunkHeader) UnmarshalBinary(bin []byte) error { + hdr.typ = ResType(btou16(bin)) + if !hdr.typ.IsSupported() { + return fmt.Errorf("%s not supported", hdr.typ) + } + hdr.headerByteSize = btou16(bin[2:]) + hdr.byteSize = btou32(bin[4:]) + return nil +} + +func (hdr chunkHeader) MarshalBinary() ([]byte, error) { + if !hdr.typ.IsSupported() { + return nil, fmt.Errorf("%s not supported", hdr.typ) + } + bin := make([]byte, 8) + putu16(bin, uint16(hdr.typ)) + putu16(bin[2:], hdr.headerByteSize) + putu32(bin[4:], hdr.byteSize) + return bin, nil +} + +type XML struct { + chunkHeader + + Pool *Pool + Map *Map + + Namespace *Namespace + Children []*Element + + // tmp field used when unmarshalling + stack []*Element +} + +// TODO this is used strictly for querying in tests and dependent on +// current XML.UnmarshalBinary implementation. Look into moving directly +// into tests. +var debugIndices = make(map[encoding.BinaryMarshaler]int) + +func (bx *XML) UnmarshalBinary(bin []byte) error { + buf := bin + if err := (&bx.chunkHeader).UnmarshalBinary(bin); err != nil { + return err + } + buf = buf[8:] + + // TODO this is tracked strictly for querying in tests; look into moving this + // functionality directly into tests if possible. + debugIndex := 8 + + for len(buf) > 0 { + t := ResType(btou16(buf)) + k, err := bx.kind(t) + if err != nil { + return err + } + if err := k.UnmarshalBinary(buf); err != nil { + return err + } + debugIndices[k.(encoding.BinaryMarshaler)] = debugIndex + debugIndex += int(k.size()) + buf = buf[k.size():] + } + return nil +} + +func (bx *XML) kind(t ResType) (unmarshaler, error) { + switch t { + case ResStringPool: + if bx.Pool != nil { + return nil, fmt.Errorf("pool already exists") + } + bx.Pool = new(Pool) + return bx.Pool, nil + case ResXMLResourceMap: + if bx.Map != nil { + return nil, fmt.Errorf("resource map already exists") + } + bx.Map = new(Map) + return bx.Map, nil + case ResXMLStartNamespace: + if bx.Namespace != nil { + return nil, fmt.Errorf("namespace start already exists") + } + bx.Namespace = new(Namespace) + return bx.Namespace, nil + case ResXMLEndNamespace: + if bx.Namespace.end != nil { + return nil, fmt.Errorf("namespace end already exists") + } + bx.Namespace.end = new(Namespace) + return bx.Namespace.end, nil + case ResXMLStartElement: + el := new(Element) + if len(bx.stack) == 0 { + bx.Children = append(bx.Children, el) + } else { + n := len(bx.stack) + var p *Element + p, bx.stack = bx.stack[n-1], bx.stack[:n-1] + p.Children = append(p.Children, el) + bx.stack = append(bx.stack, p) + } + bx.stack = append(bx.stack, el) + return el, nil + case ResXMLEndElement: + n := len(bx.stack) + var el *Element + el, bx.stack = bx.stack[n-1], bx.stack[:n-1] + if el.end != nil { + return nil, fmt.Errorf("element end already exists") + } + el.end = new(ElementEnd) + return el.end, nil + case ResXMLCharData: // TODO + cdt := new(CharData) + el := bx.stack[len(bx.stack)-1] + if el.head == nil { + el.head = cdt + } else if el.tail == nil { + el.tail = cdt + } else { + return nil, fmt.Errorf("element head and tail already contain chardata") + } + return cdt, nil + default: + return nil, fmt.Errorf("unexpected type %s", t) + } +} + +func (bx *XML) MarshalBinary() ([]byte, error) { + var ( + bin, b []byte + err error + ) + b, err = bx.chunkHeader.MarshalBinary() + if err != nil { + return nil, err + } + bin = append(bin, b...) + + b, err = bx.Pool.MarshalBinary() + if err != nil { + return nil, err + } + bin = append(bin, b...) + + b, err = bx.Map.MarshalBinary() + if err != nil { + return nil, err + } + bin = append(bin, b...) + + b, err = bx.Namespace.MarshalBinary() + if err != nil { + return nil, err + } + bin = append(bin, b...) + + for _, child := range bx.Children { + if err := marshalRecurse(child, &bin); err != nil { + return nil, err + } + } + + b, err = bx.Namespace.end.MarshalBinary() + if err != nil { + return nil, err + } + bin = append(bin, b...) + + return bin, nil +} + +func marshalRecurse(el *Element, bin *[]byte) error { + b, err := el.MarshalBinary() + if err != nil { + return err + } + *bin = append(*bin, b...) + + if el.head != nil { + b, err := el.head.MarshalBinary() + if err != nil { + return err + } + *bin = append(*bin, b...) + } + + for _, child := range el.Children { + if err := marshalRecurse(child, bin); err != nil { + return err + } + } + + b, err = el.end.MarshalBinary() + if err != nil { + return err + } + *bin = append(*bin, b...) + + return nil +} + +func (bx *XML) iterElements() <-chan *Element { + ch := make(chan *Element, 1) + go func() { + for _, el := range bx.Children { + iterElementsRecurse(el, ch) + } + close(ch) + }() + return ch +} + +func iterElementsRecurse(el *Element, ch chan *Element) { + ch <- el + for _, e := range el.Children { + iterElementsRecurse(e, ch) + } +} diff --git a/internal/binres/binres_string.go b/internal/binres/binres_string.go new file mode 100644 index 0000000..d1d08a8 --- /dev/null +++ b/internal/binres/binres_string.go @@ -0,0 +1,87 @@ +// generated by stringer -output binres_string.go -type ResType,DataType; DO NOT EDIT + +package binres + +import "fmt" + +const ( + _ResType_name_0 = "ResNullResStringPoolResTableResXML" + _ResType_name_1 = "ResXMLStartNamespaceResXMLEndNamespaceResXMLStartElementResXMLEndElementResXMLCharData" + _ResType_name_2 = "ResXMLResourceMap" + _ResType_name_3 = "ResTablePackageResTableTypeResTableTypeSpec" +) + +var ( + _ResType_index_0 = [...]uint8{7, 20, 28, 34} + _ResType_index_1 = [...]uint8{20, 38, 56, 72, 86} + _ResType_index_2 = [...]uint8{17} + _ResType_index_3 = [...]uint8{15, 27, 43} +) + +func (i ResType) String() string { + switch { + case 0 <= i && i <= 3: + lo := uint8(0) + if i > 0 { + lo = _ResType_index_0[i-1] + } + return _ResType_name_0[lo:_ResType_index_0[i]] + case 256 <= i && i <= 260: + i -= 256 + lo := uint8(0) + if i > 0 { + lo = _ResType_index_1[i-1] + } + return _ResType_name_1[lo:_ResType_index_1[i]] + case i == 384: + return _ResType_name_2 + case 512 <= i && i <= 514: + i -= 512 + lo := uint8(0) + if i > 0 { + lo = _ResType_index_3[i-1] + } + return _ResType_name_3[lo:_ResType_index_3[i]] + default: + return fmt.Sprintf("ResType(%d)", i) + } +} + +const ( + _DataType_name_0 = "DataNullDataReferenceDataAttributeDataStringDataFloatDataDimensionDataFractionDataDynamicReference" + _DataType_name_1 = "DataIntDecDataIntHexDataIntBool" + _DataType_name_2 = "DataIntColorARGB8DataIntColorRGB8DataIntColorARGB4DataIntColorRGB4" +) + +var ( + _DataType_index_0 = [...]uint8{8, 21, 34, 44, 53, 66, 78, 98} + _DataType_index_1 = [...]uint8{10, 20, 31} + _DataType_index_2 = [...]uint8{17, 33, 50, 66} +) + +func (i DataType) String() string { + switch { + case 0 <= i && i <= 7: + lo := uint8(0) + if i > 0 { + lo = _DataType_index_0[i-1] + } + return _DataType_name_0[lo:_DataType_index_0[i]] + case 16 <= i && i <= 18: + i -= 16 + lo := uint8(0) + if i > 0 { + lo = _DataType_index_1[i-1] + } + return _DataType_name_1[lo:_DataType_index_1[i]] + case 28 <= i && i <= 31: + i -= 28 + lo := uint8(0) + if i > 0 { + lo = _DataType_index_2[i-1] + } + return _DataType_name_2[lo:_DataType_index_2[i]] + default: + return fmt.Sprintf("DataType(%d)", i) + } +} diff --git a/internal/binres/binres_test.go b/internal/binres/binres_test.go new file mode 100644 index 0000000..739e1e0 --- /dev/null +++ b/internal/binres/binres_test.go @@ -0,0 +1,271 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binres + +import ( + "archive/zip" + "bytes" + "encoding" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "os" + "sort" + "strings" + "testing" +) + +func printrecurse(t *testing.T, pl *Pool, el *Element, ws string) { + for _, attr := range el.attrs { + ns := "" + if attr.NS != math.MaxUint32 { + ns = pl.strings[int(attr.NS)] + nss := strings.Split(ns, "/") + ns = nss[len(nss)-1] + } + + val := "" + if attr.RawValue != math.MaxUint32 { + val = pl.strings[int(attr.RawValue)] + } else { + switch attr.TypedValue.Type { + case DataIntDec: + val = fmt.Sprintf("%v", attr.TypedValue.Value) + case DataIntBool: + val = fmt.Sprintf("%v", attr.TypedValue.Value == 1) + default: + val = fmt.Sprintf("0x%08X", attr.TypedValue.Value) + } + } + dt := attr.TypedValue.Type + + t.Logf("%s|attr:ns(%v) name(%s) val(%s) valtyp(%s)\n", ws, ns, pl.strings[int(attr.Name)], val, dt) + } + t.Log() + for _, e := range el.Children { + printrecurse(t, pl, e, ws+" ") + } +} + +func TestBootstrap(t *testing.T) { + bin, err := ioutil.ReadFile("testdata/bootstrap.bin") + if err != nil { + log.Fatal(err) + } + + checkMarshal := func(res encoding.BinaryMarshaler, bsize int) { + b, err := res.MarshalBinary() + if err != nil { + t.Error(err) + } + idx := debugIndices[res] + a := bin[idx : idx+bsize] + if !bytes.Equal(a, b) { + x, y := len(a), len(b) + if x != y { + t.Errorf("%v: %T: byte length does not match, have %v, want %v", idx, res, y, x) + } + if x > y { + x, y = y, x + } + mismatch := false + for i := 0; i < x; i++ { + if mismatch = a[i] != b[i]; mismatch { + t.Errorf("%v: %T: first byte mismatch at %v of %v", idx, res, i, bsize) + break + } + } + if mismatch { + // print out a reasonable amount of data to help identify issues + truncate := x > 1300 + if truncate { + x = 1300 + } + t.Log(" HAVE WANT") + for i := 0; i < x; i += 4 { + he, we := 4, 4 + if i+he >= x { + he = x - i + } + if i+we >= y { + we = y - i + } + t.Logf("%3v | % X % X\n", i, b[i:i+he], a[i:i+we]) + } + if truncate { + t.Log("... output truncated.") + } + } + } + } + + bxml := new(XML) + if err := bxml.UnmarshalBinary(bin); err != nil { + t.Fatal(err) + } + + for i, x := range bxml.Pool.strings { + t.Logf("Pool(%v): %q\n", i, x) + } + + for _, e := range bxml.Children { + printrecurse(t, bxml.Pool, e, "") + } + + checkMarshal(&bxml.chunkHeader, int(bxml.headerByteSize)) + checkMarshal(bxml.Pool, bxml.Pool.size()) + checkMarshal(bxml.Map, bxml.Map.size()) + checkMarshal(bxml.Namespace, bxml.Namespace.size()) + + for el := range bxml.iterElements() { + checkMarshal(el, el.size()) + checkMarshal(el.end, el.end.size()) + } + + checkMarshal(bxml.Namespace.end, bxml.Namespace.end.size()) + checkMarshal(bxml, bxml.size()) +} + +func retset(xs []string) []string { + m := make(map[string]struct{}) + fo := xs[:0] + for _, x := range xs { + if x == "" { + continue + } + if _, ok := m[x]; !ok { + m[x] = struct{}{} + fo = append(fo, x) + } + } + return fo +} + +type ByNamespace []xml.Attr + +func (a ByNamespace) Len() int { return len(a) } +func (a ByNamespace) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByNamespace) Less(i, j int) bool { + if a[j].Name.Space == "" { + return a[i].Name.Space != "" + } + return false +} + +// WIP approximation of first steps to be taken to encode manifest +func TestEncode(t *testing.T) { + f, err := os.Open("testdata/bootstrap.xml") + if err != nil { + t.Fatal(err) + } + + var attrs []xml.Attr + + dec := xml.NewDecoder(f) + for { + tkn, err := dec.Token() + if err != nil { + if err == io.EOF { + break + } + return + // t.Fatal(err) + } + tkn = xml.CopyToken(tkn) + + switch tkn := tkn.(type) { + case xml.StartElement: + attrs = append(attrs, tkn.Attr...) + default: + // t.Error("unhandled token type", tkn) + } + } + + bvc := xml.Attr{ + Name: xml.Name{ + Space: "", + Local: "platformBuildVersionCode", + }, + Value: "10", + } + bvn := xml.Attr{ + Name: xml.Name{ + Space: "", + Local: "platformBuildVersionName", + }, + Value: "2.3.3", + } + attrs = append(attrs, bvc, bvn) + + sort.Sort(ByNamespace(attrs)) + var names, vals []string + for _, attr := range attrs { + if strings.HasSuffix(attr.Name.Space, "tools") { + continue + } + names = append(names, attr.Name.Local) + vals = append(vals, attr.Value) + } + + var all []string + all = append(all, names...) + all = append(all, vals...) + + // do not eliminate duplicates until the entire slice has been composed. + // consider + // all attribute names come first followed by values; in such a case, the value "label" + // would be a reference to the same "android:label" in the string pool which will occur + // within the beginning of the pool where other attr names are located. + pl := new(Pool) + for _, x := range retset(all) { + pl.strings = append(pl.strings, x) + // t.Logf("Pool(%v) %q\n", i, x) + } +} + +func TestAndroidJar(t *testing.T) { + zr, err := zip.OpenReader("/home/daniel/local/android-sdk/platforms/android-10/android.jar") + if err != nil { + t.Fatal(err) + } + defer zr.Close() + + buf := new(bytes.Buffer) + for _, f := range zr.File { + if f.Name == "resources.arsc" { + rc, err := f.Open() + if err != nil { + t.Fatal(err) + } + _, err = io.Copy(buf, rc) + if err != nil { + t.Fatal(err) + } + rc.Close() + break + } + } + if buf.Len() == 0 { + t.Fatal("failed to read resources.arsc") + } + + bin := buf.Bytes() + + tbl := &Table{} + if err := tbl.UnmarshalBinary(bin); err != nil { + t.Fatal(err) + } + + t.Logf("%+v\n", tbl.TableHeader) + t.Logf("%+v\n", tbl.pool.chunkHeader) + for i, x := range tbl.pool.strings[:10] { + t.Logf("pool(%v) %s\n", i, x) + } + + t.Logf("%+v\n", tbl.pkg) +} diff --git a/internal/binres/data.go b/internal/binres/data.go new file mode 100644 index 0000000..483aa14 --- /dev/null +++ b/internal/binres/data.go @@ -0,0 +1,50 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binres + +type DataType uint8 + +// explicitly defined for clarity and resolvability with apt source +const ( + DataNull DataType = 0x00 // either 0 or 1 for resource undefined or empty + DataReference DataType = 0x01 // ResTable_ref, a reference to another resource table entry + DataAttribute DataType = 0x02 // attribute resource identifier + DataString DataType = 0x03 // index into the containing resource table's global value string pool + DataFloat DataType = 0x04 // single-precision floating point number + DataDimension DataType = 0x05 // complex number encoding a dimension value, such as "100in" + DataFraction DataType = 0x06 // complex number encoding a fraction of a container + DataDynamicReference DataType = 0x07 // dynamic ResTable_ref, which needs to be resolved before it can be used like a TYPE_REFERENCE. + DataIntDec DataType = 0x10 // raw integer value of the form n..n + DataIntHex DataType = 0x11 // raw integer value of the form 0xn..n + DataIntBool DataType = 0x12 // either 0 or 1, for input "false" or "true" + DataIntColorARGB8 DataType = 0x1c // raw integer value of the form #aarrggbb + DataIntColorRGB8 DataType = 0x1d // raw integer value of the form #rrggbb + DataIntColorARGB4 DataType = 0x1e // raw integer value of the form #argb + DataIntColorRGB4 DataType = 0x1f // raw integer value of the form #rgb +) + +type Data struct { + ByteSize uint16 + Res0 uint8 // always 0, useful for debugging bad read offsets + Type DataType + Value uint32 +} + +func (d *Data) UnmarshalBinary(bin []byte) error { + d.ByteSize = btou16(bin) + d.Res0 = uint8(bin[2]) + d.Type = DataType(bin[3]) + d.Value = btou32(bin[4:]) + return nil +} + +func (d *Data) MarshalBinary() ([]byte, error) { + bin := make([]byte, 8) + putu16(bin, d.ByteSize) + bin[2] = byte(d.Res0) + bin[3] = byte(d.Type) + putu32(bin[4:], d.Value) + return bin, nil +} diff --git a/internal/binres/node.go b/internal/binres/node.go new file mode 100644 index 0000000..81d4156 --- /dev/null +++ b/internal/binres/node.go @@ -0,0 +1,225 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binres + +// NodeHeader is header all xml node types have, providing additional +// information regarding an xml node over binChunkHeader. +type NodeHeader struct { + chunkHeader + LineNumber uint32 // line number in source file this element appears + Comment PoolRef // optional xml comment associated with element, MaxUint32 if none +} + +func (hdr *NodeHeader) UnmarshalBinary(bin []byte) error { + if err := (&hdr.chunkHeader).UnmarshalBinary(bin); err != nil { + return err + } + hdr.LineNumber = btou32(bin[8:]) + hdr.Comment = PoolRef(btou32(bin[12:])) + return nil +} + +func (hdr *NodeHeader) MarshalBinary() ([]byte, error) { + bin := make([]byte, 16) + b, err := hdr.chunkHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + putu32(bin[8:], hdr.LineNumber) + putu32(bin[12:], uint32(hdr.Comment)) + return bin, nil +} + +type Namespace struct { + NodeHeader + prefix PoolRef + uri PoolRef + + end *Namespace // TODO don't let this type be recursive +} + +func (ns *Namespace) UnmarshalBinary(bin []byte) error { + if err := (&ns.NodeHeader).UnmarshalBinary(bin); err != nil { + return err + } + buf := bin[ns.headerByteSize:] + ns.prefix = PoolRef(btou32(buf)) + ns.uri = PoolRef(btou32(buf[4:])) + return nil +} + +func (ns *Namespace) MarshalBinary() ([]byte, error) { + bin := make([]byte, 24) + b, err := ns.NodeHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + putu32(bin[16:], uint32(ns.prefix)) + putu32(bin[20:], uint32(ns.uri)) + return bin, nil +} + +type Element struct { + NodeHeader + NS PoolRef + Name PoolRef // name of node if element, otherwise chardata if CDATA + AttributeStart uint16 // byte offset where attrs start + AttributeSize uint16 // byte size of attrs + AttributeCount uint16 // length of attrs + IdIndex uint16 // Index (1-based) of the "id" attribute. 0 if none. + ClassIndex uint16 // Index (1-based) of the "class" attribute. 0 if none. + StyleIndex uint16 // Index (1-based) of the "style" attribute. 0 if none. + + attrs []*Attribute + Children []*Element + end *ElementEnd + + head, tail *CharData +} + +func (el *Element) UnmarshalBinary(bin []byte) error { + if err := (&el.NodeHeader).UnmarshalBinary(bin); err != nil { + return err + } + buf := bin[el.headerByteSize:] // 16 + el.NS = PoolRef(btou32(buf)) + el.Name = PoolRef(btou32(buf[4:])) + el.AttributeStart = btou16(buf[8:]) + el.AttributeSize = btou16(buf[10:]) + el.AttributeCount = btou16(buf[12:]) + el.IdIndex = btou16(buf[14:]) + el.ClassIndex = btou16(buf[16:]) + el.StyleIndex = btou16(buf[18:]) + + buf = buf[el.AttributeStart:] + el.attrs = make([]*Attribute, int(el.AttributeCount)) + for i := range el.attrs { + attr := new(Attribute) + if err := attr.UnmarshalBinary(buf); err != nil { + return err + } + el.attrs[i] = attr + buf = buf[el.AttributeSize:] + } + + return nil +} + +func (el *Element) MarshalBinary() ([]byte, error) { + bin := make([]byte, 16+20+len(el.attrs)*int(el.AttributeSize)) + b, err := el.NodeHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + putu32(bin[16:], uint32(el.NS)) + putu32(bin[20:], uint32(el.Name)) + putu16(bin[24:], el.AttributeStart) + putu16(bin[26:], el.AttributeSize) + putu16(bin[28:], el.AttributeCount) + putu16(bin[30:], el.IdIndex) + putu16(bin[32:], el.ClassIndex) + putu16(bin[34:], el.StyleIndex) + + buf := bin[36:] + for _, attr := range el.attrs { + b, err := attr.MarshalBinary() + if err != nil { + return nil, err + } + copy(buf, b) + buf = buf[int(el.AttributeSize):] + } + + return bin, nil +} + +// ElementEnd marks the end of an element node, either Element or CharData. +type ElementEnd struct { + NodeHeader + NS PoolRef + Name PoolRef // name of node if binElement, raw chardata if binCharData +} + +func (el *ElementEnd) UnmarshalBinary(bin []byte) error { + (&el.NodeHeader).UnmarshalBinary(bin) + buf := bin[el.headerByteSize:] + el.NS = PoolRef(btou32(buf)) + el.Name = PoolRef(btou32(buf[4:])) + return nil +} + +func (el *ElementEnd) MarshalBinary() ([]byte, error) { + bin := make([]byte, 24) + b, err := el.NodeHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + putu32(bin[16:], uint32(el.NS)) + putu32(bin[20:], uint32(el.Name)) + return bin, nil +} + +type Attribute struct { + NS PoolRef + Name PoolRef + RawValue PoolRef // The original raw string value of this attribute. + TypedValue Data // Processesd typed value of this attribute. +} + +func (attr *Attribute) UnmarshalBinary(bin []byte) error { + attr.NS = PoolRef(btou32(bin)) + attr.Name = PoolRef(btou32(bin[4:])) + attr.RawValue = PoolRef(btou32(bin[8:])) + return (&attr.TypedValue).UnmarshalBinary(bin[12:]) +} + +func (attr *Attribute) MarshalBinary() ([]byte, error) { + bin := make([]byte, 20) + putu32(bin, uint32(attr.NS)) + putu32(bin[4:], uint32(attr.Name)) + putu32(bin[8:], uint32(attr.RawValue)) + b, err := attr.TypedValue.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin[12:], b) + return bin, nil +} + +// CharData represents a CDATA node and includes ref to node's text value. +type CharData struct { + NodeHeader + RawData PoolRef // raw character data + TypedData Data // typed value of character data +} + +func (cdt *CharData) UnmarshalBinary(bin []byte) error { + if err := (&cdt.NodeHeader).UnmarshalBinary(bin); err != nil { + return err + } + buf := bin[cdt.headerByteSize:] + cdt.RawData = PoolRef(btou32(buf)) + return (&cdt.TypedData).UnmarshalBinary(buf[4:]) +} + +func (cdt *CharData) MarshalBinary() ([]byte, error) { + bin := make([]byte, 28) + b, err := cdt.NodeHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + putu32(bin[16:], uint32(cdt.RawData)) + b, err = cdt.TypedData.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin[20:], b) + return bin, nil +} diff --git a/internal/binres/package.go b/internal/binres/package.go new file mode 100644 index 0000000..8b967dd --- /dev/null +++ b/internal/binres/package.go @@ -0,0 +1,101 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binres + +type TableHeader struct { + chunkHeader + packageCount uint32 +} + +func (hdr *TableHeader) UnmarshalBinary(bin []byte) error { + if err := (&hdr.chunkHeader).UnmarshalBinary(bin); err != nil { + return err + } + hdr.packageCount = btou32(bin[8:]) + return nil +} + +func (hdr TableHeader) MarshalBinary() ([]byte, error) { + bin := make([]byte, 12) + b, err := hdr.chunkHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + putu32(b[8:], hdr.packageCount) + return bin, nil +} + +// TODO next up: package chunk +// https://justanapplication.wordpress.com/2011/09/16/ +type Table struct { + TableHeader + pool *Pool + pkg *Package +} + +func (tbl *Table) UnmarshalBinary(bin []byte) error { + buf := bin + if err := (&tbl.TableHeader).UnmarshalBinary(buf); err != nil { + return err + } + buf = buf[tbl.headerByteSize:] + tbl.pool = new(Pool) + if err := tbl.pool.UnmarshalBinary(buf); err != nil { + return err + } + buf = buf[tbl.pool.size():] + tbl.pkg = new(Package) + if err := tbl.pkg.UnmarshalBinary(buf); err != nil { + return err + } + return nil +} + +type Package struct { + chunkHeader + + // If this is a base package, its ID. Package IDs start + // at 1 (corresponding to the value of the package bits in a + // resource identifier). 0 means this is not a base package. + id uint32 + + // name of package, zero terminated + name [128]uint16 + + // Offset to a ResStringPool_header defining the resource + // type symbol table. If zero, this package is inheriting from + // another base package (overriding specific values in it). + typeStrings uint32 + + // Last index into typeStrings that is for public use by others. + lastPublicType uint32 + + // Offset to a ResStringPool_header defining the resource + // key symbol table. If zero, this package is inheriting from + // another base package (overriding specific values in it). + keyStrings uint32 + + // Last index into keyStrings that is for public use by others. + lastPublicKey uint32 + + typeIdOffset uint32 +} + +func (pkg *Package) UnmarshalBinary(bin []byte) error { + if err := (&pkg.chunkHeader).UnmarshalBinary(bin); err != nil { + return err + } + pkg.id = btou32(bin[8:]) + for i := range pkg.name { + pkg.name[i] = btou16(bin[12+i*2:]) + } + pkg.typeStrings = btou32(bin[140:]) + pkg.lastPublicType = btou32(bin[144:]) + pkg.keyStrings = btou32(bin[148:]) + pkg.lastPublicKey = btou32(bin[152:]) + pkg.typeIdOffset = btou32(bin[156:]) + return nil +} diff --git a/internal/binres/pool.go b/internal/binres/pool.go new file mode 100644 index 0000000..2e2d113 --- /dev/null +++ b/internal/binres/pool.go @@ -0,0 +1,276 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binres + +import ( + "fmt" + "unicode/utf16" +) + +const ( + SortedFlag uint32 = 1 << 0 + UTF8Flag = 1 << 8 +) + +// PoolRef is the i'th string in a pool. +type PoolRef uint32 + +// Pool has the following structure marshalled: +// +// binChunkHeader +// StringCount uint32 // Number of strings in this pool +// StyleCount uint32 // Number of style spans in pool +// Flags uint32 // SortedFlag, UTF8Flag +// StringsStart uint32 // Index of string data from header +// StylesStart uint32 // Index of style data from header +// +// StringIndices []uint32 // starting at zero +// +// // UTF16 entries are concatenations of the following: +// // [2]byte uint16 string length, exclusive +// // [2]byte [optional] low word if high bit of length was set +// // [n]byte data +// // [2]byte 0x0000 terminator +// Strings []uint16 +type Pool struct { + chunkHeader + + strings []string + styles []*Span + flags uint32 // SortedFlag, UTF8Flag +} + +func (pl *Pool) IsSorted() bool { return pl.flags&SortedFlag == SortedFlag } +func (pl *Pool) IsUTF8() bool { return pl.flags&UTF8Flag == UTF8Flag } + +func (pl *Pool) UnmarshalBinary(bin []byte) error { + if err := (&pl.chunkHeader).UnmarshalBinary(bin); err != nil { + return err + } + if pl.typ != ResStringPool { + return fmt.Errorf("have type %s, want %s", pl.typ, ResStringPool) + } + + nstrings := btou32(bin[8:]) + pl.strings = make([]string, nstrings) + + nstyles := btou32(bin[12:]) + pl.styles = make([]*Span, nstyles) + + pl.flags = btou32(bin[16:]) + + offstrings := btou32(bin[20:]) + offstyle := btou32(bin[24:]) + + hdrlen := 28 // header size is always 28 + if pl.IsUTF8() { + for i := range pl.strings { + ii := btou32(bin[hdrlen+i*4:]) // index of string index + r := int(offstrings + ii) // read index of string + + // for char and byte sizes below, if leading bit set, + // treat first as 7-bit high word and the next byte as low word. + + // get char size of string + cn := uint8(bin[r]) + rcn := int(cn) + r++ + if cn&(1<<7) != 0 { + cn0 := int(cn ^ (1 << 7)) // high word + cn1 := int(bin[r]) // low word + rcn = cn0*256 + cn1 + r++ + } + + // get byte size of string + // TODO(d) i've seen at least one case in android.jar resource table that has only + // highbit set, effectively making 7-bit highword zero. The reason for this is currently + // unknown but would make tests that unmarshal-marshal to match bytes impossible to pass. + // The values noted were high-word: 0 (after highbit unset), low-word: 141 + // I don't recall character count but was around ?78? + // The case here may very well be that the size is treated like int8 triggering the use of + // two bytes to store size even though the implementation uses uint8. + n := uint8(bin[r]) + r++ + rn := int(n) + if n&(1<<7) != 0 { + n0 := int(n ^ (1 << 7)) // high word + n1 := int(bin[r]) // low word + rn = n0*(1<<8) + n1 + r++ + } + + // + data := bin[r : r+rn] + if x := uint8(bin[r+rn]); x != 0 { + return fmt.Errorf("expected zero terminator, got %v for byte size %v char len %v", x, rn, rcn) + } + pl.strings[i] = string(data) + } + } else { + for i := range pl.strings { + ii := btou32(bin[hdrlen+i*4:]) // index of string index + r := int(offstrings + ii) // read index of string + n := btou16(bin[r:]) // string length + rn := int(n) + r += 2 + + if n&(1<<15) != 0 { // TODO this is untested + n0 := int(n ^ (1 << 15)) // high word + n1 := int(btou16(bin[r:])) + rn = n0*(1<<16) + n1 + r += 2 + } + + data := make([]uint16, int(rn)) + for i := range data { + data[i] = btou16(bin[r+(2*i):]) + } + + r += int(n * 2) + if x := btou16(bin[r:]); x != 0 { + return fmt.Errorf("expected zero terminator, got 0x%04X\n%s", x) + } + + pl.strings[i] = string(utf16.Decode(data)) + } + } + + // TODO + _ = offstyle + // styii := hdrlen + int(nstrings*4) + // for i := range pl.styles { + // ii := btou32(bin[styii+i*4:]) + // r := int(offstyle + ii) + // spn := new(binSpan) + // spn.UnmarshalBinary(bin[r:]) + // pl.styles[i] = spn + // } + + return nil +} + +func (pl *Pool) MarshalBinary() ([]byte, error) { + if pl.IsUTF8() { + return nil, fmt.Errorf("encode utf8 not supported") + } + + var ( + hdrlen = 28 + // indices of string indices + iis = make([]uint32, len(pl.strings)) + iislen = len(iis) * 4 + // utf16 encoded strings concatenated together + strs []uint16 + ) + for i, x := range pl.strings { + if len(x)>>16 > 0 { + panic(fmt.Errorf("string lengths over 1<<15 not yet supported, got len %d", len(x))) + } + p := utf16.Encode([]rune(x)) + if len(p) == 0 { + strs = append(strs, 0x0000, 0x0000) + } else { + strs = append(strs, uint16(len(p))) // string length (implicitly includes zero terminator to follow) + strs = append(strs, p...) + strs = append(strs, 0) // zero terminated + } + // indices start at zero + if i+1 != len(iis) { + iis[i+1] = uint32(len(strs) * 2) // utf16 byte index + } + } + + // check strings is 4-byte aligned, pad with zeros if not. + for x := (len(strs) * 2) % 4; x != 0; x -= 2 { + strs = append(strs, 0x0000) + } + + strslen := len(strs) * 2 + + hdr := chunkHeader{ + typ: ResStringPool, + headerByteSize: 28, + byteSize: uint32(28 + iislen + strslen), + } + + bin := make([]byte, hdr.byteSize) + + hdrbin, err := hdr.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, hdrbin) + + putu32(bin[8:], uint32(len(pl.strings))) + putu32(bin[12:], uint32(len(pl.styles))) + putu32(bin[16:], pl.flags) + putu32(bin[20:], uint32(hdrlen+iislen)) + putu32(bin[24:], 0) // index of styles start, is 0 when styles length is 0 + + buf := bin[28:] + for _, x := range iis { + putu32(buf, x) + buf = buf[4:] + } + for _, x := range strs { + putu16(buf, x) + buf = buf[2:] + } + + if len(buf) != 0 { + panic(fmt.Errorf("failed to fill allocated buffer, %v bytes left over", len(buf))) + } + + return bin, nil +} + +type Span struct { + name PoolRef + + firstChar, lastChar uint32 +} + +func (spn *Span) UnmarshalBinary(bin []byte) error { + const end = 0xFFFFFFFF + spn.name = PoolRef(btou32(bin)) + if spn.name == end { + return nil + } + spn.firstChar = btou32(bin[4:]) + spn.lastChar = btou32(bin[8:]) + return nil +} + +// Map contains a uint32 slice mapping strings in the string +// pool back to resource identifiers. The i'th element of the slice +// is also the same i'th element of the string pool. +type Map struct { + chunkHeader + rs []uint32 +} + +func (m *Map) UnmarshalBinary(bin []byte) error { + (&m.chunkHeader).UnmarshalBinary(bin) + buf := bin[m.headerByteSize:m.byteSize] + m.rs = make([]uint32, len(buf)/4) + for i := range m.rs { + m.rs[i] = btou32(buf[i*4:]) + } + return nil +} + +func (m *Map) MarshalBinary() ([]byte, error) { + bin := make([]byte, 8+len(m.rs)*4) + b, err := m.chunkHeader.MarshalBinary() + if err != nil { + return nil, err + } + copy(bin, b) + for i, r := range m.rs { + putu32(bin[8+i*4:], r) + } + return bin, nil +} diff --git a/internal/binres/testdata/bootstrap.bin b/internal/binres/testdata/bootstrap.bin new file mode 100644 index 0000000000000000000000000000000000000000..ada309b7dc1698dcc8a2887a1cf63cdc0e4c4736 GIT binary patch literal 2188 zcma)-zfV(96vxkNDJ{RGKt%;_yeX=W^-T(tbk`A1bg5o_yrC?A#HXU ztbitX2fl-SFk~I8kwM$)*0QE8^IJu8d}qzdw(WLCkacVyW4UEdkNBHqtbt_(Z3C99 zy=2vn@eVrvitXAak1QLGwGT!gA!8wawJ6>g{u|_2<7wJMW_33Uu@9rX%|@_M1uxKS z5>IiP%xZWms}Ac1QB_;k?NwoFVO>SDfPKibW4G)Ik0N$6Q1+#SeaXez#;%OV7E;-@ z-MR^1j~8m9?O?ZoUD@Shb{37g6q0QV>utM3W_5JcL_d4y@&7)(lgh!YF4T#m{6exB zk8&^BTXaoyyRO0tmJObk>!Ml8uG$jz8M{XP*B!Wz&RSfHK z>*U(%t~k^Uom>Zm_6CkAbn(uq{ZI$HeXenzbbr*rBGyMMJj#{yDA_7}PvBd&dv@1-jm+K$ zIrnXvP6fdX7zTkM!6--rzHq?>Fbn3j4)WY+-E*@;#t?ECq|ooh^x7L^UEs+eHDsru zJwBcJ1?JKbgs!VnOG|416mmS$NlG`0oapNE>acWqWTC4|ODd0>Ncn4D<1};Sp*R{Y zU!?Yw#{lzmR9DhZeCzBhw(K<~nTwB4svF|d{0vYHlG5?F;A5x+r&BM)f!(;I@0)Vh zT0{L!_RaSjzaUGtKBoR(6jM^VesC_*NlM4QkC|5wq*Kj`#opLw=E`FgDIe)IrkN`b z{#M-Z{4=Z+Q&Kve#ra4l>3f;vQVji1D%Wm=!@~>A6~m7{Y{ctrW|=F04f*SAweS6l z%;g_HTlL@~P>zy5pQNAXQ_h}mj`>M^mm)q%&zJP`de!9lH1~Sd;`KUXuP?^*x?5ha R+`YZtrGWb7=k@;G`v>4?j5z=R literal 0 HcmV?d00001 diff --git a/internal/binres/testdata/bootstrap.xml b/internal/binres/testdata/bootstrap.xml new file mode 100644 index 0000000..e50dbe6 --- /dev/null +++ b/internal/binres/testdata/bootstrap.xml @@ -0,0 +1,33 @@ + + + + + + + + + + here is some text + + + + + + diff --git a/internal/binres/testdata/gen.sh b/internal/binres/testdata/gen.sh new file mode 100755 index 0000000..b92daf2 --- /dev/null +++ b/internal/binres/testdata/gen.sh @@ -0,0 +1,15 @@ +#! /usr/bin/sh + +# version of build-tools tests run against +AAPT=${ANDROID_HOME}/build-tools/23.0.1/aapt + +# minimum version of android api for resource identifiers supported +APIJAR=${ANDROID_HOME}/platforms/android-10/android.jar + +for f in *.xml; do + cp "$f" AndroidManifest.xml + "$AAPT" p -M AndroidManifest.xml -I "$APIJAR" -F tmp.apk + unzip -qq -o tmp.apk + mv AndroidManifest.xml "${f:0:-3}bin" + rm tmp.apk +done