feat(message): added waku message deterministic hashing

waku/persistence/db_key.go

@@ -4,7 +4,7 @@ import (
 	"encoding/binary"
 	"errors"
 
-	"github.com/waku-org/go-waku/waku/v2/utils"
+	"github.com/waku-org/go-waku/waku/v2/hash"
 )
 
 const (
@@ -35,7 +35,7 @@ func (k *DBKey) Bytes() []byte {
 
 // NewDBKey creates a new DBKey with the given values.
 func NewDBKey(senderTimestamp uint64, receiverTimestamp uint64, pubsubTopic string, digest []byte) *DBKey {
-	pubSubHash := utils.SHA256([]byte(pubsubTopic))
+	pubSubHash := hash.SHA256([]byte(pubsubTopic))
 
 	var k DBKey
 	k.raw = make([]byte, DBKeyLength)

waku/v2/hash/hash.go

@@ -1,4 +1,4 @@
-package utils
+package hash
 
 import (
 	"crypto/sha256"
@@ -10,13 +10,15 @@ var sha256Pool = sync.Pool{New: func() interface{} {
 	return sha256.New()
 }}
 
-func SHA256(data []byte) []byte {
+func SHA256(data ...[]byte) []byte {
 	h, ok := sha256Pool.Get().(hash.Hash)
 	if !ok {
 		h = sha256.New()
 	}
 	defer sha256Pool.Put(h)
 	h.Reset()
-	h.Write(data)
+	for i := range data {
+		h.Write(data[i])
+	}
 	return h.Sum(nil)
 }

waku/v2/node/wakunode2.go

@@ -560,7 +560,7 @@ func (w *WakuNode) Publish(ctx context.Context, msg *pb.WakuMessage) error {
 		return errors.New("cannot publish message, relay and lightpush are disabled")
 	}
 
-	hash, _, _ := msg.Hash()
+	hash := msg.Hash(relay.DefaultWakuTopic)
 	err := try.Do(func(attempt int) (bool, error) {
 		var err error
 

waku/v2/protocol/envelope.go

@@ -1,9 +1,9 @@
 package protocol
 
 import (
+	"github.com/waku-org/go-waku/waku/v2/hash"
 	wpb "github.com/waku-org/go-waku/waku/v2/protocol/pb"
 	"github.com/waku-org/go-waku/waku/v2/protocol/store/pb"
-	"github.com/waku-org/go-waku/waku/v2/utils"
 )
 
 // Envelope contains information about the pubsub topic of a WakuMessage
@@ -11,7 +11,6 @@ import (
 // protobuffer
 type Envelope struct {
 	msg   *wpb.WakuMessage
-	size  int
 	hash  []byte
 	index *pb.Index
 }
@@ -20,11 +19,10 @@ type Envelope struct {
 // It's used as a way to know to which Pubsub topic belongs a WakuMessage
 // as well as generating a hash based on the bytes that compose the message
 func NewEnvelope(msg *wpb.WakuMessage, receiverTime int64, pubSubTopic string) *Envelope {
-	messageHash, dataLen, _ := msg.Hash()
-	hash := utils.SHA256(append([]byte(msg.ContentTopic), msg.Payload...))
+	messageHash := msg.Hash(pubSubTopic)
+	hash := hash.SHA256([]byte(msg.ContentTopic), msg.Payload)
 	return &Envelope{
 		msg:  msg,
-		size: dataLen,
 		hash: messageHash,
 		index: &pb.Index{
 			Digest:       hash[:],
@@ -50,11 +48,6 @@ func (e *Envelope) Hash() []byte {
 	return e.hash
 }
 
-// Size returns the byte size of the WakuMessage
-func (e *Envelope) Size() int {
-	return e.size
-}
-
 func (env *Envelope) Index() *pb.Index {
 	return env.index
 }

waku/v2/protocol/envelope_test.go

@@ -1,6 +1,7 @@
 package protocol
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -20,14 +21,12 @@ func TestEnvelope(t *testing.T) {
 
 	topic := e.PubsubTopic()
 	require.Equal(t, "test", topic)
-
 	hash := e.Hash()
+	fmt.Println(hash)
+
 	require.Equal(
 		t,
-		[]uint8([]byte{0xc7, 0xaf, 0xc3, 0xe9, 0x9, 0xd1, 0xc6, 0xb4, 0x81, 0xb3, 0xdf, 0x4f, 0x16, 0x1a, 0xe4, 0xc9, 0x9c, 0x8, 0x4e, 0x5, 0xe4, 0xeb, 0x5f, 0x9b, 0x58, 0xb5, 0xf4, 0xde, 0xe9, 0x73, 0x18, 0x7b}),
+		[]uint8{70, 218, 246, 174, 188, 127, 199, 220, 111, 30, 61, 218, 238, 60, 83, 3, 179, 98, 85, 35, 7, 107, 188, 138, 32, 70, 170, 126, 55, 21, 71, 70},
 		hash,
 	)
-
-	size := e.Size()
-	require.Equal(t, 14, size)
 }

waku/v2/protocol/lightpush/waku_lightpush.go

@@ -216,7 +216,7 @@ func (wakuLP *WakuLightPush) PublishToTopic(ctx context.Context, message *wpb.Wa
 	}
 
 	if response.IsSuccess {
-		hash, _, _ := message.Hash()
+		hash := message.Hash(topic)
 		wakuLP.log.Info("waku.lightpush published", logging.HexString("hash", hash))
 		return hash, nil
 	} else {

waku/v2/protocol/pb/utils.go

@@ -1,23 +1,8 @@
 package pb
 
-import (
-	"crypto/sha256"
-
-	proto "google.golang.org/protobuf/proto"
-)
+import "github.com/waku-org/go-waku/waku/v2/hash"
 
 // Hash calculates the hash of a waku message
-func (msg *WakuMessage) Hash() ([]byte, int, error) {
-	out, err := proto.Marshal(msg)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	return Hash(out), len(out), nil
-}
-
-// Hash calculates a hash from a byte slice using sha2-256 for the hashing algorithm
-func Hash(data []byte) []byte {
-	hash := sha256.Sum256(data)
-	return hash[:]
+func (msg *WakuMessage) Hash(pubsubTopic string) []byte {
+	return hash.SHA256([]byte(pubsubTopic), msg.Payload, []byte(msg.ContentTopic), msg.Meta)
 }

waku/v2/protocol/pb/utils_test.go

@@ -1,14 +1,16 @@
 package pb
 
 import (
+	"encoding/hex"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"github.com/waku-org/go-waku/waku/v2/hash"
 )
 
 func TestHash(t *testing.T) {
 	expected := []byte{0xa5, 0x91, 0xa6, 0xd4, 0xb, 0xf4, 0x20, 0x40, 0x4a, 0x1, 0x17, 0x33, 0xcf, 0xb7, 0xb1, 0x90, 0xd6, 0x2c, 0x65, 0xbf, 0xb, 0xcd, 0xa3, 0x2b, 0x57, 0xb2, 0x77, 0xd9, 0xad, 0x9f, 0x14, 0x6e}
-	result := Hash([]byte("Hello World"))
+	result := hash.SHA256([]byte("Hello World"))
 	require.Equal(t, expected, result)
 }
 
@@ -19,8 +21,48 @@ func TestEnvelopeHash(t *testing.T) {
 	msg.Timestamp = 123456789123456789
 	msg.Version = 1
 
-	expected := []byte{0xf8, 0xa4, 0x70, 0x32, 0x76, 0xf2, 0x36, 0x99, 0x24, 0xc5, 0xfe, 0x36, 0xee, 0x3a, 0x96, 0xcb, 0xca, 0x45, 0x54, 0xed, 0xd3, 0x1a, 0x19, 0xb1, 0x68, 0xac, 0x49, 0xe9, 0x26, 0xed, 0xb6, 0xc6}
-	result, _, err := msg.Hash()
-	require.NoError(t, err)
+	expected := []byte{0xee, 0xcf, 0xf5, 0xb7, 0xdd, 0x54, 0x2d, 0x68, 0x9e, 0x7d, 0x64, 0xa3, 0xb8, 0x50, 0x8b, 0xba, 0xc, 0xf1, 0xac, 0xb6, 0xf7, 0x1c, 0x9f, 0xf2, 0x32, 0x7, 0x5b, 0xfd, 0x90, 0x5c, 0xe5, 0xa1}
+	result := msg.Hash("test")
 	require.Equal(t, expected, result)
 }
+
+func TestEmptyMeta(t *testing.T) {
+	pubsubTopic := "/waku/2/default-waku/proto"
+	msg := new(WakuMessage)
+	msg.ContentTopic = "/waku/2/default-content/proto"
+	msg.Payload = []byte("\x01\x02\x03\x04TEST\x05\x06\x07\x08")
+
+	msg.Meta = []byte{}
+	msg.Timestamp = 123456789123456789
+	msg.Version = 1
+
+	messageHash := msg.Hash(pubsubTopic)
+
+	require.Equal(t, "87619d05e563521d9126749b45bd4cc2430df0607e77e23572d874ed9c1aaa62", hex.EncodeToString(messageHash))
+}
+
+func Test13ByteMeta(t *testing.T) {
+	pubsubTopic := "/waku/2/default-waku/proto"
+	msg := new(WakuMessage)
+	msg.ContentTopic = "/waku/2/default-content/proto"
+	msg.Payload = []byte("\x01\x02\x03\x04TEST\x05\x06\x07\x08")
+	msg.Meta = []byte("\x73\x75\x70\x65\x72\x2d\x73\x65\x63\x72\x65\x74")
+	msg.Version = 1
+
+	messageHash := msg.Hash(pubsubTopic)
+
+	require.Equal(t, "4fdde1099c9f77f6dae8147b6b3179aba1fc8e14a7bf35203fc253ee479f135f", hex.EncodeToString(messageHash))
+}
+
+func TestZeroLenPayload(t *testing.T) {
+	pubsubTopic := "/waku/2/default-waku/proto"
+	msg := new(WakuMessage)
+	msg.ContentTopic = "/waku/2/default-content/proto"
+	msg.Payload = []byte{}
+	msg.Meta = []byte("\x73\x75\x70\x65\x72\x2d\x73\x65\x63\x72\x65\x74")
+	msg.Version = 1
+
+	messageHash := msg.Hash(pubsubTopic)
+
+	require.Equal(t, "e1a9596237dbe2cc8aaf4b838c46a7052df6bc0d42ba214b998a8bfdbe8487d6", hex.EncodeToString(messageHash))
+}

waku/v2/protocol/relay/waku_relay.go

@@ -18,11 +18,11 @@ import (
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 	"github.com/waku-org/go-waku/logging"
 	v2 "github.com/waku-org/go-waku/waku/v2"
+	"github.com/waku-org/go-waku/waku/v2/hash"
 	"github.com/waku-org/go-waku/waku/v2/metrics"
 	waku_proto "github.com/waku-org/go-waku/waku/v2/protocol"
 	"github.com/waku-org/go-waku/waku/v2/protocol/pb"
 	"github.com/waku-org/go-waku/waku/v2/timesource"
-	"github.com/waku-org/go-waku/waku/v2/utils"
 )
 
 const WakuRelayID_v200 = protocol.ID("/vac/waku/relay/2.0.0")
@@ -52,7 +52,7 @@ type WakuRelay struct {
 }
 
 func msgIdFn(pmsg *pubsub_pb.Message) string {
-	return string(utils.SHA256(pmsg.Data))
+	return string(hash.SHA256(pmsg.Data))
 }
 
 // NewWakuRelay returns a new instance of a WakuRelay struct
@@ -190,7 +190,7 @@ func (w *WakuRelay) PublishToTopic(ctx context.Context, message *pb.WakuMessage,
 		return nil, err
 	}
 
-	hash := pb.Hash(out)
+	hash := message.Hash(topic)
 
 	w.log.Debug("waku.relay published", zap.String("hash", hex.EncodeToString(hash)))
 

waku/v2/protocol/store/waku_store_client.go

@@ -314,13 +314,6 @@ func (store *WakuStore) Query(ctx context.Context, query Query, opts ...HistoryR
 		return nil, errors.New("invalid cursor")
 	}
 
-	var messageIDs [][]byte
-	for _, m := range response.Messages {
-		messageID, _, _ := m.Hash()
-		messageIDs = append(messageIDs, messageID)
-	}
-	store.log.Info("waku.store retrieved", logging.HexArray("hashes", messageIDs))
-
 	result := &Result{
 		store:    store,
 		Messages: response.Messages,