internal/binres: encode manifest correctly for packaging

This fixes remaining issues that prevented use of package
to encode manifest for packaging correctly.

* Chunk headers are now encoded based on content
* Fixes for namespace and value fields
* Improved sorting based on native aapt output

Change-Id: Ic63046973a7b0431533463ed4dd2de50f1d73191
Reviewed-on: https://go-review.googlesource.com/19224
Reviewed-by: David Crawshaw <crawshaw@golang.org>

internal/binres/binres.go

@@ -63,14 +63,7 @@ func errWrongType(have ResType, want ...ResType) error {
 type ResType uint16
 
 func (t ResType) IsSupported() bool {
-	// explicit for clarity
+	return t != ResNull
-	return t == ResStringPool || t == ResXML ||
-		t == ResXMLStartNamespace || t == ResXMLEndNamespace ||
-		t == ResXMLStartElement || t == ResXMLEndElement ||
-		t == ResXMLCharData ||
-		t == ResXMLResourceMap ||
-		t == ResTable || t == ResTablePackage ||
-		t == ResTableTypeSpec || t == ResTableType
 }
 
 // explicitly defined for clarity and resolvability with apt source
@@ -235,7 +228,7 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 							Space: "",
 							Local: "platformBuildVersionName",
 						},
-						Value: "4.0.3",
+						Value: "4.0.4-1406430",
 					})
 			}
 
@@ -267,6 +260,7 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 				el.attrs = append(el.attrs, nattr)
 
 				if attr.Name.Space == "" {
+					nattr.NS = NoEntry
 					// TODO it's unclear how to query these
 					switch attr.Name.Local {
 					case "platformBuildVersionCode":
@@ -306,6 +300,7 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 							// TODO identify 0x3e, in bootstrap.xml this is the native lib name
 							nattr.RawValue = pool.ref(attr.Value)
 							nattr.TypedValue.Type = DataString
+							nattr.TypedValue.Value = uint32(nattr.RawValue)
 						case DataIntBool, DataType(0x08):
 							nattr.TypedValue.Type = DataIntBool
 							switch attr.Value {
@@ -398,13 +393,30 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 				}
 			}
 		case xml.EndElement:
+			if tkn.Name.Local == "manifest" {
+				bx.Namespace.end = &Namespace{
+					NodeHeader: NodeHeader{
+						LineNumber: uint32(line),
+						Comment:    NoEntry,
+					},
+					prefix: 0,
+					uri:    0,
+				}
+			}
 			n := len(bx.stack)
 			var el *Element
 			el, bx.stack = bx.stack[n-1], bx.stack[:n-1]
 			if el.end != nil {
 				return nil, fmt.Errorf("element end already exists")
 			}
-			el.end = new(ElementEnd)
+			el.end = &ElementEnd{
+				NodeHeader: NodeHeader{
+					LineNumber: uint32(line),
+					Comment:    NoEntry,
+				},
+				NS:   el.NS,
+				Name: el.Name,
+			}
 		case xml.Comment, xml.ProcInst:
 			// discard
 		default:
@@ -424,6 +436,9 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 	var arecurse func(*Element)
 	arecurse = func(el *Element) {
 		for _, attr := range el.attrs {
+			if attr.NS == NoEntry {
+				continue
+			}
 			if attr.NS.Resolve(pool) == androidSchema {
 				bx.Pool.strings = append(bx.Pool.strings, attr.Name.Resolve(pool))
 			}
@@ -448,7 +463,7 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 	var brecurse func(*Element)
 	brecurse = func(el *Element) {
 		for _, attr := range el.attrs {
-			if attr.NS.Resolve(pool) == "" {
+			if attr.NS == NoEntry {
 				bx.Pool.strings = append(bx.Pool.strings, attr.Name.Resolve(pool))
 			}
 		}
@@ -458,7 +473,7 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 		for _, attr := range el.attrs {
 			if attr.RawValue != NoEntry {
 				bx.Pool.strings = append(bx.Pool.strings, attr.RawValue.Resolve(pool))
-			} else if attr.NS.Resolve(pool) == "" {
+			} else if attr.NS == NoEntry {
 				bx.Pool.strings = append(bx.Pool.strings, fmt.Sprintf("%+v", attr.TypedValue.Value))
 			}
 		}
@@ -502,8 +517,10 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 	resolve = func(el *Element) {
 		if el.NS != NoEntry {
 			el.NS = bx.Pool.ref(el.NS.Resolve(pool))
+			el.end.NS = el.NS
 		}
 		el.Name = bx.Pool.ref(el.Name.Resolve(pool))
+		el.end.Name = el.Name
 		for _, attr := range el.attrs {
 			if attr.NS != NoEntry {
 				attr.NS = bx.Pool.ref(attr.NS.Resolve(pool))
@@ -511,6 +528,9 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 			attr.Name = bx.Pool.ref(attr.Name.Resolve(pool))
 			if attr.RawValue != NoEntry {
 				attr.RawValue = bx.Pool.ref(attr.RawValue.Resolve(pool))
+				if attr.TypedValue.Type == DataString {
+					attr.TypedValue.Value = uint32(attr.RawValue)
+				}
 			}
 		}
 		for _, child := range el.Children {
@@ -524,8 +544,8 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 	var asort func(*Element)
 	asort = func(el *Element) {
 		sort.Sort(byType(el.attrs))
-		// sort.Sort(byName(el.attrs))
 		sort.Sort(byNamespace(el.attrs))
+		sort.Sort(byName(el.attrs))
 		for _, child := range el.Children {
 			asort(child)
 		}
@@ -534,6 +554,17 @@ func UnmarshalXML(r io.Reader) (*XML, error) {
 		asort(el)
 	}
 
+	for i, s := range bx.Pool.strings {
+		switch s {
+		case androidSchema:
+			bx.Namespace.uri = PoolRef(i)
+			bx.Namespace.end.uri = PoolRef(i)
+		case "android":
+			bx.Namespace.prefix = PoolRef(i)
+			bx.Namespace.end.prefix = PoolRef(i)
+		}
+	}
+
 	return bx, nil
 }
 
@@ -629,6 +660,9 @@ func (bx *XML) kind(t ResType) (unmarshaler, error) {
 }
 
 func (bx *XML) MarshalBinary() ([]byte, error) {
+	bx.typ = ResXML
+	bx.headerByteSize = 8
+
 	var (
 		bin, b []byte
 		err    error
@@ -669,6 +703,7 @@ func (bx *XML) MarshalBinary() ([]byte, error) {
 	}
 	bin = append(bin, b...)
 
+	putu32(bin[4:], uint32(len(bin)))
 	return bin, nil
 }
 
@@ -788,7 +823,9 @@ type byName []*Attribute
 
 func (a byName) Len() int { return len(a) }
 func (a byName) Less(i, j int) bool {
-	return a[i].TypedValue.Type == a[j].TypedValue.Type && a[i].Name < a[j].Name
+	return (a[i].TypedValue.Type == DataString || a[i].TypedValue.Type == DataIntDec) &&
+		(a[j].TypedValue.Type == DataString || a[j].TypedValue.Type == DataIntDec) &&
+		a[i].Name < a[j].Name
 }
 func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
 

internal/binres/binres_test.go

@@ -13,7 +13,6 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"testing"
 )
@@ -53,6 +52,53 @@ func printrecurse(t *testing.T, pl *Pool, el *Element, ws string) {
 	}
 }
 
+func compareBytes(a, b []byte) error {
+	if bytes.Equal(a, b) {
+		return nil
+	}
+	buf := new(bytes.Buffer)
+	x, y := len(a), len(b)
+	if x != y {
+		fmt.Fprintf(buf, "byte length does not match, have %v, want %v\n", y, x)
+	}
+	if x > y {
+		x, y = y, x
+	}
+	mismatch := false
+	for i := 0; i < x; i++ {
+		if mismatch = a[i] != b[i]; mismatch {
+			fmt.Fprintf(buf, "first byte mismatch at %v\n", i)
+			break
+		}
+	}
+	if mismatch {
+		// print out a reasonable amount of data to help identify issues
+		truncate := x > 3300
+		if truncate {
+			x = 3300
+		}
+		buf.WriteString("      HAVE               WANT\n")
+		for i := 0; i < x; i += 4 {
+			he, we := 4, 4
+			if i+he >= x {
+				he = x - i
+			}
+			if i+we >= y {
+				we = y - i
+			}
+			notequal := ""
+			if !bytes.Equal(b[i:i+he], a[i:i+we]) {
+				notequal = "***"
+			}
+			fmt.Fprintf(buf, "%3v | % X        % X    %s\n", i, b[i:i+he], a[i:i+we], notequal)
+		}
+		if truncate {
+			fmt.Fprint(buf, "... output truncated.\n")
+		}
+	}
+	return fmt.Errorf(buf.String())
+}
+
 func TestBootstrap(t *testing.T) {
 	bin, err := ioutil.ReadFile("testdata/bootstrap.bin")
 	if err != nil {
@@ -143,7 +189,6 @@ func TestEncode(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	//
 	bin, err := ioutil.ReadFile("testdata/bootstrap.bin")
 	if err != nil {
 		log.Fatal(err)
@@ -161,17 +206,75 @@ func TestEncode(t *testing.T) {
 		t.Error(err)
 	}
 
-	if !reflect.DeepEqual(bxml.Namespace, bx.Namespace) {
+	if err := compareNamespaces(bx.Namespace, bxml.Namespace); err != nil {
-		t.Errorf("Namespace doesn't match, have %+v, want %+v", bx.Namespace, bxml.Namespace)
+		t.Error(err)
 	}
 
-	if !reflect.DeepEqual(bxml.Children, bx.Children) {
+	if err := compareElements(bx, bxml); err != nil {
-		t.Error("Children don't match")
+		t.Fatal(err)
 	}
 
-	for _, el := range bx.Children {
+	have, err := bx.MarshalBinary()
-		printrecurse(t, bx.Pool, el, "")
+	if err != nil {
+		t.Fatal(err)
 	}
+	if err := compareBytes(bin, have); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func compareElements(have, want *XML) error {
+	h, w := have.iterElements(), want.iterElements()
+	buf := new(bytes.Buffer)
+	for {
+		a, b := <-h, <-w
+		if a == nil || b == nil {
+			break
+		}
+		if a.NS != b.NS ||
+			a.Name != b.Name {
+			return fmt.Errorf("elements don't match, have %+v, want %+v", a, b)
+		}
+		if a.end.NS != b.end.NS ||
+			a.end.Name != b.end.Name {
+			return fmt.Errorf("element ends don't match, have %+v, want %+v", a.end, b.end)
+		}
+		if len(a.attrs) != len(b.attrs) {
+			return fmt.Errorf("element attribute lengths don't match, have %v, want %v", len(a.attrs), len(b.attrs))
+		}
+
+		for i, attr := range a.attrs {
+			bttr := b.attrs[i]
+			if attr.NS != bttr.NS ||
+				attr.Name != bttr.Name ||
+				attr.RawValue != bttr.RawValue ||
+				attr.TypedValue.Type != bttr.TypedValue.Type ||
+				attr.TypedValue.Value != bttr.TypedValue.Value {
+				fmt.Fprintf(buf, "attrs don't match\nhave: %+v\nwant: %+v\n", attr, bttr)
+			}
+		}
+		if buf.Len() > 0 {
+			buf.WriteString("-------------\n")
+		}
+	}
+	if buf.Len() > 0 {
+		return fmt.Errorf(buf.String())
+	}
+	return nil
+}
+
+func compareNamespaces(have, want *Namespace) error {
+	if have == nil || want == nil ||
+		have.LineNumber != want.LineNumber ||
+		have.Comment != want.Comment ||
+		have.prefix != want.prefix ||
+		have.uri != want.uri {
+		return fmt.Errorf("namespaces don't match, have %+v, want %+v", have, want)
+	}
+	if have.end != nil || want.end != nil {
+		return compareNamespaces(have.end, want.end)
+	}
+	return nil
 }
 
 func rtou(a []TableRef) (b []uint32) {

internal/binres/node.go

@@ -52,7 +52,15 @@ func (ns *Namespace) UnmarshalBinary(bin []byte) error {
 }
 
 func (ns *Namespace) MarshalBinary() ([]byte, error) {
-	bin := make([]byte, 24)
+	if ns.end == nil {
+		ns.typ = ResXMLEndNamespace
+	} else {
+		ns.typ = ResXMLStartNamespace
+	}
+	ns.headerByteSize = 16
+	ns.byteSize = 24
+
+	bin := make([]byte, ns.byteSize)
 	b, err := ns.NodeHeader.MarshalBinary()
 	if err != nil {
 		return nil, err
@@ -88,6 +96,7 @@ func (el *Element) UnmarshalBinary(buf []byte) error {
 	buf = buf[el.headerByteSize:]
 	el.NS = PoolRef(btou32(buf))
 	el.Name = PoolRef(btou32(buf[4:]))
+
 	el.AttributeStart = btou16(buf[8:])
 	el.AttributeSize = btou16(buf[10:])
 	el.AttributeCount = btou16(buf[12:])
@@ -110,7 +119,17 @@ func (el *Element) UnmarshalBinary(buf []byte) error {
 }
 
 func (el *Element) MarshalBinary() ([]byte, error) {
-	bin := make([]byte, 16+20+len(el.attrs)*int(el.AttributeSize))
+	el.typ = ResXMLStartElement
+	el.headerByteSize = 16
+	el.AttributeSize = 20
+	el.AttributeStart = 20
+	el.AttributeCount = uint16(len(el.attrs))
+	el.IdIndex = 0
+	el.ClassIndex = 0
+	el.StyleIndex = 0
+	el.byteSize = uint32(el.headerByteSize) + uint32(el.AttributeStart) + uint32(len(el.attrs)*int(el.AttributeSize))
+
+	bin := make([]byte, el.byteSize)
 	b, err := el.NodeHeader.MarshalBinary()
 	if err != nil {
 		return nil, err
@@ -154,6 +173,10 @@ func (el *ElementEnd) UnmarshalBinary(bin []byte) error {
 }
 
 func (el *ElementEnd) MarshalBinary() ([]byte, error) {
+	el.typ = ResXMLEndElement
+	el.headerByteSize = 16
+	el.byteSize = 24
+
 	bin := make([]byte, 24)
 	b, err := el.NodeHeader.MarshalBinary()
 	if err != nil {
@@ -209,7 +232,11 @@ func (cdt *CharData) UnmarshalBinary(bin []byte) error {
 }
 
 func (cdt *CharData) MarshalBinary() ([]byte, error) {
-	bin := make([]byte, 28)
+	cdt.typ = ResXMLCharData
+	cdt.headerByteSize = 16
+	cdt.byteSize = 28
+
+	bin := make([]byte, cdt.byteSize)
 	b, err := cdt.NodeHeader.MarshalBinary()
 	if err != nil {
 		return nil, err

internal/binres/pool.go

@@ -215,6 +215,7 @@ func (pl *Pool) MarshalBinary() ([]byte, error) {
 		putu32(buf, x)
 		buf = buf[4:]
 	}
+
 	for _, x := range strs {
 		putu16(buf, x)
 		buf = buf[2:]
@@ -263,7 +264,10 @@ func (m *Map) UnmarshalBinary(bin []byte) error {
 }
 
 func (m *Map) MarshalBinary() ([]byte, error) {
-	bin := make([]byte, 8+len(m.rs)*4)
+	m.typ = ResXMLResourceMap
+	m.headerByteSize = 8
+	m.byteSize = uint32(m.headerByteSize) + uint32(len(m.rs)*4)
+	bin := make([]byte, m.byteSize)
 	b, err := m.chunkHeader.MarshalBinary()
 	if err != nil {
 		return nil, err

internal/binres/table.go

@@ -687,7 +687,7 @@ func (d *Data) UnmarshalBinary(bin []byte) error {
 
 func (d *Data) MarshalBinary() ([]byte, error) {
 	bin := make([]byte, 8)
-	putu16(bin, d.ByteSize)
+	putu16(bin, 8)
 	bin[2] = byte(d.Res0)
 	bin[3] = byte(d.Type)
 	putu32(bin[4:], d.Value)

internal/binres/testdata/bootstrap.xml

@@ -11,7 +11,7 @@ license that can be found in the LICENSE file.
 	android:versionName=""
 	xmlns:tools="http://schemas.android.com/tools">
 
-	<uses-sdk android:minSdkVersion="9" />
+	<uses-sdk android:minSdkVersion="15" />
 
 	<uses-permission android:name="android.permission.INTERNET" />