package common // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package binary implements simple translation between numbers and byte // sequences and encoding and decoding of varints. // // Numbers are translated by reading and writing fixed-size values. // A fixed-size value is either a fixed-size arithmetic // type (int8, uint8, int16, float32, complex64, ...) // or an array or struct containing only fixed-size values. // // The varint functions encode and decode single integer values using // a variable-length encoding; smaller values require fewer bytes. // For a specification, see // http://code.google.com/apis/protocolbuffers/docs/encoding.html. // // This package favors simplicity over efficiency. Clients that require // high-performance serialization, especially for large data structures, // should look at more advanced solutions such as the encoding/gob // package or protocol buffers. import ( "errors" "io" "math" "reflect" ) // A ByteOrder specifies how to convert byte sequences into // 16-, 32-, or 64-bit unsigned integers. type ByteOrder interface { Uint16([]byte) uint16 Uint32([]byte) uint32 Uint64([]byte) uint64 PutUint16([]byte, uint16) PutUint32([]byte, uint32) PutUint64([]byte, uint64) String() string } // LittleEndian is the little-endian implementation of ByteOrder. var LittleEndian littleEndian // BigEndian is the big-endian implementation of ByteOrder. var BigEndian bigEndian type littleEndian struct{} func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 } func (littleEndian) PutUint16(b []byte, v uint16) { b[0] = byte(v) b[1] = byte(v >> 8) } func (littleEndian) Uint32(b []byte) uint32 { return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 } func (littleEndian) PutUint32(b []byte, v uint32) { b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) b[3] = byte(v >> 24) } func (littleEndian) Uint64(b []byte) uint64 { return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 } func (littleEndian) PutUint64(b []byte, v uint64) { b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) b[3] = byte(v >> 24) b[4] = byte(v >> 32) b[5] = byte(v >> 40) b[6] = byte(v >> 48) b[7] = byte(v >> 56) } func (littleEndian) String() string { return "LittleEndian" } func (littleEndian) GoString() string { return "binary.LittleEndian" } type bigEndian struct{} func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 } func (bigEndian) PutUint16(b []byte, v uint16) { b[0] = byte(v >> 8) b[1] = byte(v) } func (bigEndian) Uint32(b []byte) uint32 { return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 } func (bigEndian) PutUint32(b []byte, v uint32) { b[0] = byte(v >> 24) b[1] = byte(v >> 16) b[2] = byte(v >> 8) b[3] = byte(v) } func (bigEndian) Uint64(b []byte) uint64 { return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } func (bigEndian) PutUint64(b []byte, v uint64) { b[0] = byte(v >> 56) b[1] = byte(v >> 48) b[2] = byte(v >> 40) b[3] = byte(v >> 32) b[4] = byte(v >> 24) b[5] = byte(v >> 16) b[6] = byte(v >> 8) b[7] = byte(v) } func (bigEndian) String() string { return "BigEndian" } func (bigEndian) GoString() string { return "binary.BigEndian" } // Read reads structured binary data from r into data. // Data must be a pointer to a fixed-size value or a slice // of fixed-size values. // Bytes read from r are decoded using the specified byte order // and written to successive fields of the data. // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. // When reading into a struct, all non-blank fields must be exported. func Read(r io.Reader, order ByteOrder, data interface{}) error { // Fast path for basic types and slices. if n := intDataSize(data); n != 0 { var b [8]byte var bs []byte if n > len(b) { bs = make([]byte, n) } else { bs = b[:n] } if _, err := io.ReadFull(r, bs); err != nil { return err } switch data := data.(type) { case *int8: *data = int8(b[0]) case *uint8: *data = b[0] case *int16: *data = int16(order.Uint16(bs)) case *uint16: *data = order.Uint16(bs) case *int32: *data = int32(order.Uint32(bs)) case *uint32: *data = order.Uint32(bs) case *int64: *data = int64(order.Uint64(bs)) case *uint64: *data = order.Uint64(bs) case []int8: for i, x := range bs { // Easier to loop over the input for 8-bit values. data[i] = int8(x) } case []uint8: copy(data, bs) case []int16: for i := range data { data[i] = int16(order.Uint16(bs[2*i:])) } case []uint16: for i := range data { data[i] = order.Uint16(bs[2*i:]) } case []int32: for i := range data { data[i] = int32(order.Uint32(bs[4*i:])) } case []uint32: for i := range data { data[i] = order.Uint32(bs[4*i:]) } case []int64: for i := range data { data[i] = int64(order.Uint64(bs[8*i:])) } case []uint64: for i := range data { data[i] = order.Uint64(bs[8*i:]) } } return nil } // Fallback to reflect-based decoding. v := reflect.ValueOf(data) size := -1 switch v.Kind() { case reflect.Ptr: v = v.Elem() size = dataSize(v) case reflect.Slice: size = dataSize(v) } if size < 0 { return errors.New("binary.Read: invalid type " + reflect.TypeOf(data).String()) } d := &decoder{order: order, buf: make([]byte, size)} if _, err := io.ReadFull(r, d.buf); err != nil { return err } d.value(v) return nil } // Write writes the binary representation of data into w. // Data must be a fixed-size value or a slice of fixed-size // values, or a pointer to such data. // Bytes written to w are encoded using the specified byte order // and read from successive fields of the data. // When writing structs, zero values are written for fields // with blank (_) field names. func Write(w io.Writer, order ByteOrder, data interface{}) error { // Fast path for basic types and slices. if n := intDataSize(data); n != 0 { var b [8]byte var bs []byte if n > len(b) { bs = make([]byte, n) } else { bs = b[:n] } switch v := data.(type) { case *int8: bs = b[:1] b[0] = byte(*v) case int8: bs = b[:1] b[0] = byte(v) case []int8: for i, x := range v { bs[i] = byte(x) } case *uint8: bs = b[:1] b[0] = *v case uint8: bs = b[:1] b[0] = byte(v) case []uint8: bs = v case *int16: bs = b[:2] order.PutUint16(bs, uint16(*v)) case int16: bs = b[:2] order.PutUint16(bs, uint16(v)) case []int16: for i, x := range v { order.PutUint16(bs[2*i:], uint16(x)) } case *uint16: bs = b[:2] order.PutUint16(bs, *v) case uint16: bs = b[:2] order.PutUint16(bs, v) case []uint16: for i, x := range v { order.PutUint16(bs[2*i:], x) } case *int32: bs = b[:4] order.PutUint32(bs, uint32(*v)) case int32: bs = b[:4] order.PutUint32(bs, uint32(v)) case []int32: for i, x := range v { order.PutUint32(bs[4*i:], uint32(x)) } case *uint32: bs = b[:4] order.PutUint32(bs, *v) case uint32: bs = b[:4] order.PutUint32(bs, v) case []uint32: for i, x := range v { order.PutUint32(bs[4*i:], x) } case *int64: bs = b[:8] order.PutUint64(bs, uint64(*v)) case int64: bs = b[:8] order.PutUint64(bs, uint64(v)) case []int64: for i, x := range v { order.PutUint64(bs[8*i:], uint64(x)) } case *uint64: bs = b[:8] order.PutUint64(bs, *v) case uint64: bs = b[:8] order.PutUint64(bs, v) case []uint64: for i, x := range v { order.PutUint64(bs[8*i:], x) } } _, err := w.Write(bs) return err } // Fallback to reflect-based encoding. v := reflect.Indirect(reflect.ValueOf(data)) size := dataSize(v) if size < 0 { return errors.New("binary.Write: invalid type " + reflect.TypeOf(data).String()) } buf := make([]byte, size) e := &encoder{order: order, buf: buf} e.value(v) _, err := w.Write(buf) return err } // Size returns how many bytes Write would generate to encode the value v, which // must be a fixed-size value or a slice of fixed-size values, or a pointer to such data. // If v is neither of these, Size returns -1. func Size(v interface{}) int { return dataSize(reflect.Indirect(reflect.ValueOf(v))) } // dataSize returns the number of bytes the actual data represented by v occupies in memory. // For compound structures, it sums the sizes of the elements. Thus, for instance, for a slice // it returns the length of the slice times the element size and does not count the memory // occupied by the header. If the type of v is not acceptable, dataSize returns -1. func dataSize(v reflect.Value) int { if v.Kind() == reflect.Slice { if s := sizeof(v.Type().Elem()); s >= 0 { return s * v.Len() } return -1 } return sizeof(v.Type()) } // sizeof returns the size >= 0 of variables for the given type or -1 if the type is not acceptable. func sizeof(t reflect.Type) int { switch t.Kind() { case reflect.Array: if s := sizeof(t.Elem()); s >= 0 { return s * t.Len() } case reflect.Struct: sum := 0 for i, n := 0, t.NumField(); i < n; i++ { s := sizeof(t.Field(i).Type) if s < 0 { return -1 } sum += s } return sum case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Ptr: return int(t.Size()) } return -1 } type coder struct { order ByteOrder buf []byte } type decoder coder type encoder coder func (d *decoder) uint8() uint8 { x := d.buf[0] d.buf = d.buf[1:] return x } func (e *encoder) uint8(x uint8) { e.buf[0] = x e.buf = e.buf[1:] } func (d *decoder) uint16() uint16 { x := d.order.Uint16(d.buf[0:2]) d.buf = d.buf[2:] return x } func (e *encoder) uint16(x uint16) { e.order.PutUint16(e.buf[0:2], x) e.buf = e.buf[2:] } func (d *decoder) uint32() uint32 { x := d.order.Uint32(d.buf[0:4]) d.buf = d.buf[4:] return x } func (e *encoder) uint32(x uint32) { e.order.PutUint32(e.buf[0:4], x) e.buf = e.buf[4:] } func (d *decoder) uint64() uint64 { x := d.order.Uint64(d.buf[0:8]) d.buf = d.buf[8:] return x } func (e *encoder) uint64(x uint64) { e.order.PutUint64(e.buf[0:8], x) e.buf = e.buf[8:] } func (d *decoder) int8() int8 { return int8(d.uint8()) } func (e *encoder) int8(x int8) { e.uint8(uint8(x)) } func (d *decoder) int16() int16 { return int16(d.uint16()) } func (e *encoder) int16(x int16) { e.uint16(uint16(x)) } func (d *decoder) int32() int32 { return int32(d.uint32()) } func (e *encoder) int32(x int32) { e.uint32(uint32(x)) } func (d *decoder) int64() int64 { return int64(d.uint64()) } func (e *encoder) int64(x int64) { e.uint64(uint64(x)) } func (d *decoder) value(v reflect.Value) { switch v.Kind() { case reflect.Array: l := v.Len() for i := 0; i < l; i++ { d.value(v.Index(i)) } case reflect.Struct: t := v.Type() l := v.NumField() for i := 0; i < l; i++ { // Note: Calling v.CanSet() below is an optimization. // It would be sufficient to check the field name, // but creating the StructField info for each field is // costly (run "go test -bench=ReadStruct" and compare // results when making changes to this code). if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" { d.value(v) } else { d.skip(v) } } case reflect.Slice: l := v.Len() for i := 0; i < l; i++ { d.value(v.Index(i)) } case reflect.Int8: v.SetInt(int64(d.int8())) case reflect.Int16: v.SetInt(int64(d.int16())) case reflect.Int32: v.SetInt(int64(d.int32())) case reflect.Int64: v.SetInt(d.int64()) case reflect.Uint8: v.SetUint(uint64(d.uint8())) case reflect.Uint16: v.SetUint(uint64(d.uint16())) case reflect.Uint32: v.SetUint(uint64(d.uint32())) case reflect.Uint64: v.SetUint(d.uint64()) case reflect.Float32: v.SetFloat(float64(math.Float32frombits(d.uint32()))) case reflect.Float64: v.SetFloat(math.Float64frombits(d.uint64())) case reflect.Complex64: v.SetComplex(complex( float64(math.Float32frombits(d.uint32())), float64(math.Float32frombits(d.uint32())), )) case reflect.Complex128: v.SetComplex(complex( math.Float64frombits(d.uint64()), math.Float64frombits(d.uint64()), )) } } func (e *encoder) value(v reflect.Value) { switch v.Kind() { case reflect.Array: l := v.Len() for i := 0; i < l; i++ { e.value(v.Index(i)) } case reflect.Struct: t := v.Type() l := v.NumField() for i := 0; i < l; i++ { // see comment for corresponding code in decoder.value() if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" { e.value(v) } else { e.skip(v) } } case reflect.Slice: l := v.Len() for i := 0; i < l; i++ { e.value(v.Index(i)) } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: switch v.Type().Kind() { case reflect.Int8: e.int8(int8(v.Int())) case reflect.Int16: e.int16(int16(v.Int())) case reflect.Int32: e.int32(int32(v.Int())) case reflect.Int64: e.int64(v.Int()) } case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: switch v.Type().Kind() { case reflect.Uint8: e.uint8(uint8(v.Uint())) case reflect.Uint16: e.uint16(uint16(v.Uint())) case reflect.Uint32: e.uint32(uint32(v.Uint())) case reflect.Uint64: e.uint64(v.Uint()) } case reflect.Float32, reflect.Float64: switch v.Type().Kind() { case reflect.Float32: e.uint32(math.Float32bits(float32(v.Float()))) case reflect.Float64: e.uint64(math.Float64bits(v.Float())) } case reflect.Complex64, reflect.Complex128: switch v.Type().Kind() { case reflect.Complex64: x := v.Complex() e.uint32(math.Float32bits(float32(real(x)))) e.uint32(math.Float32bits(float32(imag(x)))) case reflect.Complex128: x := v.Complex() e.uint64(math.Float64bits(real(x))) e.uint64(math.Float64bits(imag(x))) } } } func (d *decoder) skip(v reflect.Value) { d.buf = d.buf[dataSize(v):] } func (e *encoder) skip(v reflect.Value) { n := dataSize(v) for i := range e.buf[0:n] { e.buf[i] = 0 } e.buf = e.buf[n:] } // intDataSize returns the size of the data required to represent the data when encoded. // It returns zero if the type cannot be implemented by the fast path in Read or Write. func intDataSize(data interface{}) int { switch data := data.(type) { case int8, *int8, *uint8: return 1 case []int8: return len(data) case []uint8: return len(data) case int16, *int16, *uint16: return 2 case []int16: return 2 * len(data) case []uint16: return 2 * len(data) case int32, *int32, *uint32: return 4 case []int32: return 4 * len(data) case []uint32: return 4 * len(data) case int64, *int64, *uint64: return 8 case []int64: return 8 * len(data) case []uint64: return 8 * len(data) } return 0 }