chore: filter v2 tests push valid payload (#904)

tests/string_generators_test.go

@@ -14,19 +14,15 @@ func TestStringGenerators(t *testing.T) {
 
 	// Generate string and print out to console
 	for i := 0; i < 1000; i++ {
-		x, err := GenerateRandomASCIIString(1, 4097)
+		x, err := GenerateRandomASCIIString(4097)
 		require.NoError(t, err)
 		log.Info("Generated random ASCII string", zap.String(strconv.Itoa(i), x))
 
-		x, err = GenerateRandomUTF8String(1, 4097, false)
+		x, err = GenerateRandomUTF8String(4097)
 		require.NoError(t, err)
 		log.Info("Generated random UTF8 string", zap.String(strconv.Itoa(i), x))
 
-		x, err = GenerateRandomUTF8String(1, 4097, true)
+		x, err = GenerateRandomJSONString(4099)
-		require.NoError(t, err)
-		log.Info("Generated uncommon UTF8 string", zap.String(strconv.Itoa(i), x))
-
-		x, err = GenerateRandomJSONString()
 		require.NoError(t, err)
 		log.Info("Generated random JSON string", zap.String(strconv.Itoa(i), x))
 
@@ -38,7 +34,7 @@ func TestStringGenerators(t *testing.T) {
 		require.NoError(t, err)
 		log.Info("Generated random URL encoded string", zap.String(strconv.Itoa(i), x))
 
-		x, err = GenerateRandomSQLInsert()
+		x, err = GenerateRandomSQLInsert(4096)
 		require.NoError(t, err)
 		log.Info("Generated random SQL insert string", zap.String(strconv.Itoa(i), x))
 	}

tests/utils.go

@@ -38,6 +38,8 @@ import (
 	"go.uber.org/zap"
 )
 
+type StringGenerator func(maxLength int) (string, error)
+
 // GetHostAddress returns the first listen address used by a host
 func GetHostAddress(ha host.Host) multiaddr.Multiaddr {
 	return ha.Addrs()[0]
@@ -247,12 +249,12 @@ func RandomBytes(n int) ([]byte, error) {
 	return b, nil
 }
 
-func GenerateRandomASCIIString(minLength int, maxLength int) (string, error) {
+func GenerateRandomASCIIString(maxLength int) (string, error) {
-	length, err := rand.Int(rand.Reader, big.NewInt(int64(maxLength-minLength+1)))
+	length, err := rand.Int(rand.Reader, big.NewInt(int64(maxLength)))
 	if err != nil {
 		return "", err
 	}
-	length.SetInt64(length.Int64() + int64(minLength))
+	length.SetInt64(length.Int64() + 1)
 
 	const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 	result := make([]byte, length.Int64())
@@ -267,27 +269,21 @@ func GenerateRandomASCIIString(minLength int, maxLength int) (string, error) {
 	return string(result), nil
 }
 
-func GenerateRandomUTF8String(minLength int, maxLength int, withUncommon bool) (string, error) {
+func GenerateRandomUTF8String(maxLength int) (string, error) {
-	length, err := rand.Int(rand.Reader, big.NewInt(int64(maxLength-minLength+1)))
+	length, err := rand.Int(rand.Reader, big.NewInt(int64(maxLength)))
 	if err != nil {
 		return "", err
 	}
-	length.SetInt64(length.Int64() + int64(minLength))
+	length.SetInt64(length.Int64() + 1)
 
 	var (
 		runes      []rune
 		start, end int
 	)
 
-	if withUncommon {
+	// Define unicode range
-		// Unicode range for uncommon or unprintable characters, the Private Use Area (E000–F8FF)
+	start = 0x0020 // Space character
-		start = 0xE000
+	end = 0x007F   // Tilde (~)
-		end = 0xF8FF
-	} else {
-		// Define unicode range
-		start = 0x0020 // Space character
-		end = 0x007F   // Tilde (~)
-	}
 
 	for i := 0; int64(i) < length.Int64(); i++ {
 		randNum, err := rand.Int(rand.Reader, big.NewInt(int64(end-start+1)))
@@ -304,15 +300,15 @@ func GenerateRandomUTF8String(minLength int, maxLength int, withUncommon bool) (
 	return string(runes), nil
 }
 
-func GenerateRandomJSONString() (string, error) {
+func GenerateRandomJSONString(maxLength int) (string, error) {
 	// With 5 key-value pairs
 	m := make(map[string]interface{})
 	for i := 0; i < 5; i++ {
-		key, err := GenerateRandomASCIIString(1, 20)
+		key, err := GenerateRandomASCIIString(20)
 		if err != nil {
 			return "", err
 		}
-		value, err := GenerateRandomASCIIString(1, 4097)
+		value, err := GenerateRandomASCIIString(maxLength)
 		if err != nil {
 			return "", err
 		}
@@ -332,8 +328,8 @@ func GenerateRandomJSONString() (string, error) {
 	return buf.String(), nil
 }
 
-func GenerateRandomBase64String(length int) (string, error) {
+func GenerateRandomBase64String(maxLength int) (string, error) {
-	bytes, err := RandomBytes(length)
+	bytes, err := RandomBytes(maxLength)
 	if err != nil {
 		return "", err
 	}
@@ -341,8 +337,8 @@ func GenerateRandomBase64String(length int) (string, error) {
 	return base64.StdEncoding.EncodeToString(bytes), nil
 }
 
-func GenerateRandomURLEncodedString(length int) (string, error) {
+func GenerateRandomURLEncodedString(maxLength int) (string, error) {
-	randomString, err := GenerateRandomASCIIString(1, 4097)
+	randomString, err := GenerateRandomASCIIString(maxLength)
 	if err != nil {
 		return "", err
 	}
@@ -351,9 +347,9 @@ func GenerateRandomURLEncodedString(length int) (string, error) {
 	return url.QueryEscape(randomString), nil
 }
 
-func GenerateRandomSQLInsert() (string, error) {
+func GenerateRandomSQLInsert(maxLength int) (string, error) {
 	// Random table name
-	tableName, err := GenerateRandomASCIIString(1, 10)
+	tableName, err := GenerateRandomASCIIString(10)
 	if err != nil {
 		return "", err
 	}
@@ -365,7 +361,7 @@ func GenerateRandomSQLInsert() (string, error) {
 	}
 	columnNames := make([]string, columnCount)
 	for i := 0; i < columnCount; i++ {
-		columnName, err := GenerateRandomASCIIString(1, 20)
+		columnName, err := GenerateRandomASCIIString(maxLength)
 		if err != nil {
 			return "", err
 		}
@@ -375,7 +371,7 @@ func GenerateRandomSQLInsert() (string, error) {
 	// Random values
 	values := make([]string, columnCount)
 	for i := 0; i < columnCount; i++ {
-		value, err := GenerateRandomASCIIString(1, 100)
+		value, err := GenerateRandomASCIIString(maxLength)
 		if err != nil {
 			return "", err
 		}

waku/v2/protocol/filter/filter_push_test.go

@@ -0,0 +1,143 @@
+package filter
+
+import (
+	"context"
+	"github.com/waku-org/go-waku/tests"
+	"go.uber.org/zap"
+	"strconv"
+	"time"
+)
+
+func (s *FilterTestSuite) TestValidPayloadsASCII() {
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare data
+	messages := prepareData(100, false, false, true, tests.GenerateRandomASCIIString)
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}
+
+func (s *FilterTestSuite) TestValidPayloadsUTF8() {
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare data
+	messages := prepareData(100, false, false, true, tests.GenerateRandomUTF8String)
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}
+
+func (s *FilterTestSuite) TestValidPayloadsBase64() {
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare data
+	messages := prepareData(100, false, false, true, tests.GenerateRandomBase64String)
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}
+
+func (s *FilterTestSuite) TestValidPayloadsJSON() {
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare data
+	messages := prepareData(100, false, false, true, tests.GenerateRandomJSONString)
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}
+
+func (s *FilterTestSuite) TestValidPayloadsURLEncoded() {
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare data
+	messages := prepareData(100, false, false, true, tests.GenerateRandomURLEncodedString)
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}
+
+func (s *FilterTestSuite) TestValidPayloadsSQL() {
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare data
+	messages := prepareData(100, false, false, true, tests.GenerateRandomSQLInsert)
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}
+
+func (s *FilterTestSuite) TestLargePayloadsUTF8() {
+
+	s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second)
+
+	// Subscribe
+	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
+
+	// Prepare basic data
+	messages := prepareData(10, false, false, false, nil)
+
+	// Generate large string
+	for i := range messages {
+		messages[i].payload, _ = tests.GenerateRandomUTF8String(1048576)
+		s.log.Info("Generated payload with ", zap.String("length", strconv.Itoa(len(messages[i].payload))))
+	}
+
+	// All messages should be received
+	s.waitForMessages(func() {
+		s.publishMessages(messages)
+	}, s.subDetails, messages)
+
+	_, err := s.lightNode.UnsubscribeAll(s.ctx)
+	s.Require().NoError(err)
+
+}

waku/v2/protocol/filter/filter_subscribe_test.go

@@ -95,7 +95,7 @@ func (s *FilterTestSuite) TestPubSubMultiContentTopic() {
 	// Create test context
 	s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 10 seconds
 
-	messages := prepareData(3, false, true, false)
+	messages := prepareData(3, false, true, false, nil)
 
 	// Subscribe
 	for _, m := range messages {
@@ -126,7 +126,7 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() {
 	err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1)
 	s.Require().NoError(err)
 
-	messages := prepareData(2, true, true, false)
+	messages := prepareData(2, true, true, false, nil)
 
 	// Subscribe
 	for _, m := range messages {
@@ -156,7 +156,7 @@ func (s *FilterTestSuite) TestPubSubMultiOverlapContentTopic() {
 	// Create test context
 	s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 20 seconds
 
-	messages := prepareData(10, false, true, true)
+	messages := prepareData(10, false, true, true, nil)
 
 	// Subscribe
 	for _, m := range messages {
@@ -175,7 +175,7 @@ func (s *FilterTestSuite) TestPubSubMultiOverlapContentTopic() {
 
 func (s *FilterTestSuite) TestSubscriptionRefresh() {
 
-	messages := prepareData(2, false, false, true)
+	messages := prepareData(2, false, false, true, nil)
 
 	// Initial subscribe
 	s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID())
@@ -207,7 +207,7 @@ func (s *FilterTestSuite) TestContentTopicsLimit() {
 		}
 	}
 
-	messages := prepareData(maxContentTopics+1, false, true, true)
+	messages := prepareData(maxContentTopics+1, false, true, true, nil)
 
 	// Subscribe
 	for _, m := range messages[:len(messages)-1] {
@@ -311,7 +311,7 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() {
 	// Connect node2
 	lightNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL)
 
-	messages := prepareData(2, true, true, true)
+	messages := prepareData(2, true, true, true, nil)
 
 	// Subscribe separately: light node 1 -> full node
 	contentFilter := protocol.ContentFilter{PubsubTopic: messages[0].pubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].contentTopic)}

waku/v2/protocol/filter/filter_test.go

@@ -289,12 +289,14 @@ func (s *FilterTestSuite) publishMessages(msgs []WakuMsg) {
 	}
 }
 
-func prepareData(quantity int, topics, contentTopics, payloads bool) []WakuMsg {
+func prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.StringGenerator) []WakuMsg {
 	var (
-		pubsubTopic  = "/waku/2/go/filter/test" // Has to be the same with initial s.testTopic
+		pubsubTopic     = "/waku/2/go/filter/test" // Has to be the same with initial s.testTopic
-		contentTopic = "TopicA"                 // Has to be the same with initial s.testContentTopic
+		contentTopic    = "TopicA"                 // Has to be the same with initial s.testContentTopic
-		payload      = "test_msg"
+		payload         = "test_msg"
-		messages     []WakuMsg
+		messages        []WakuMsg
+		strMaxLenght    = 4097
+		generatedString = ""
 	)
 
 	for i := 0; i < quantity; i++ {
@@ -304,16 +306,21 @@ func prepareData(quantity int, topics, contentTopics, payloads bool) []WakuMsg {
 			payload:      payload,
 		}
 
+		if sg != nil {
+			generatedString, _ = sg(strMaxLenght)
+
+		}
+
 		if topics {
-			msg.pubSubTopic = fmt.Sprintf("%s%02d", pubsubTopic, i)
+			msg.pubSubTopic = fmt.Sprintf("%s%02d%s", pubsubTopic, i, generatedString)
 		}
 
 		if contentTopics {
-			msg.contentTopic = fmt.Sprintf("%s%02d", contentTopic, i)
+			msg.contentTopic = fmt.Sprintf("%s%02d%s", contentTopic, i, generatedString)
 		}
 
 		if payloads {
-			msg.payload = fmt.Sprintf("%s%02d", payload, i)
+			msg.payload = fmt.Sprintf("%s%02d%s", payload, i, generatedString)
 		}
 
 		messages = append(messages, msg)

waku/v2/protocol/filter/filter_unsubscribe_test.go

@@ -45,7 +45,7 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() {
 
 func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() {
 
-	var messages = prepareData(3, false, true, true)
+	var messages = prepareData(3, false, true, true, nil)
 
 	// Subscribe with 3 content topics
 	for _, m := range messages {
@@ -92,7 +92,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() {
 	err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1)
 	s.Require().NoError(err)
 
-	messages := prepareData(2, true, true, true)
+	messages := prepareData(2, true, true, true, nil)
 
 	// Subscribe
 	for _, m := range messages {
@@ -134,7 +134,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() {
 
 	var messages, invalidMessages []WakuMsg
 
-	messages = prepareData(2, false, true, true)
+	messages = prepareData(2, false, true, true, nil)
 
 	// Prepare "invalid" data for unsubscribe
 	invalidMessages = append(invalidMessages,
@@ -196,7 +196,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() {
 
 func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() {
 
-	var messages = prepareData(2, false, true, true)
+	var messages = prepareData(2, false, true, true, nil)
 
 	// Subscribe with 2 content topics
 	for _, m := range messages {
@@ -233,7 +233,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() {
 	err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1)
 	s.Require().NoError(err)
 
-	messages := prepareData(2, true, true, true)
+	messages := prepareData(2, true, true, true, nil)
 
 	// Subscribe
 	for _, m := range messages {