mirror of
https://github.com/logos-messaging/message-finder.git
synced 2026-01-02 14:03:11 +00:00
initial commit
This commit is contained in:
Richard Ramos
commit
11f971b7fa
6
Makefile
Normal file
6
Makefile
Normal file
@ -0,0 +1,6 @@
|
||||
.PHONY: all build
|
||||
|
||||
tests:
|
||||
go test ./... -count 1 -v
|
||||
|
||||
all: tests
|
||||
137
go.mod
Normal file
137
go.mod
Normal file
@ -0,0 +1,137 @@
|
||||
module example.com/m
|
||||
|
||||
go 1.19
|
||||
|
||||
replace github.com/ethereum/go-ethereum v1.10.26 => github.com/status-im/go-ethereum v1.10.25-status.4
|
||||
|
||||
//replace github.com/waku-org/go-waku v0.7.0 => /home/richard/waku-org/go-waku
|
||||
|
||||
require (
|
||||
github.com/ethereum/go-ethereum v1.10.26
|
||||
github.com/libp2p/go-libp2p v0.32.2
|
||||
github.com/multiformats/go-multiaddr v0.12.0
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/waku-org/go-waku v0.8.1-0.20240115144202-0723ff928263
|
||||
go.uber.org/zap v1.26.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
|
||||
github.com/avast/retry-go/v4 v4.5.1 // indirect
|
||||
github.com/beevik/ntp v0.3.0 // indirect
|
||||
github.com/benbjohnson/clock v1.3.5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btcd v0.20.1-beta // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
|
||||
github.com/deckarep/golang-set v1.8.0 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/elastic/gosigar v0.14.2 // indirect
|
||||
github.com/flynn/noise v1.0.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/go-ole/go-ole v1.2.1 // indirect
|
||||
github.com/go-stack/stack v1.8.1 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/godbus/dbus/v5 v5.1.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
|
||||
github.com/golang-migrate/migrate/v4 v4.16.2 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/gopacket v1.1.19 // indirect
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
|
||||
github.com/huin/goupnp v1.3.0 // indirect
|
||||
github.com/ipfs/go-cid v0.4.1 // indirect
|
||||
github.com/ipfs/go-log/v2 v2.5.1 // indirect
|
||||
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
|
||||
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
|
||||
github.com/klauspost/compress v1.17.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
|
||||
github.com/koron/go-ssdp v0.0.4 // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
|
||||
github.com/libp2p/go-cidranger v1.1.0 // indirect
|
||||
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
|
||||
github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
|
||||
github.com/libp2p/go-libp2p-mplex v0.9.0 // indirect
|
||||
github.com/libp2p/go-libp2p-pubsub v0.10.0 // indirect
|
||||
github.com/libp2p/go-mplex v0.7.0 // indirect
|
||||
github.com/libp2p/go-msgio v0.3.0 // indirect
|
||||
github.com/libp2p/go-nat v0.2.0 // indirect
|
||||
github.com/libp2p/go-netroute v0.2.1 // indirect
|
||||
github.com/libp2p/go-reuseport v0.4.0 // indirect
|
||||
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
|
||||
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/miekg/dns v1.1.56 // indirect
|
||||
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
|
||||
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.1.0 // indirect
|
||||
github.com/multiformats/go-base36 v0.2.0 // indirect
|
||||
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
|
||||
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.2.0 // indirect
|
||||
github.com/multiformats/go-multicodec v0.9.0 // indirect
|
||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
||||
github.com/multiformats/go-multistream v0.5.0 // indirect
|
||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.1.0 // indirect
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_golang v1.14.0 // indirect
|
||||
github.com/prometheus/client_model v0.4.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0 // indirect
|
||||
github.com/quic-go/qpack v0.4.0 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
|
||||
github.com/quic-go/quic-go v0.39.4 // indirect
|
||||
github.com/quic-go/webtransport-go v0.6.0 // indirect
|
||||
github.com/raulk/go-watchdog v1.3.0 // indirect
|
||||
github.com/rjeczalik/notify v0.9.3 // indirect
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/status-im/status-go/extkeys v1.1.2 // indirect
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.5 // indirect
|
||||
github.com/tklauser/numcpus v0.2.2 // indirect
|
||||
github.com/waku-org/go-discover v0.0.0-20221209174356-61c833f34d98 // indirect
|
||||
github.com/waku-org/go-libp2p-rendezvous v0.0.0-20240110193335-a67d1cc760a0 // indirect
|
||||
github.com/waku-org/go-zerokit-rln v0.1.14-0.20240102145250-fa738c0bdf59 // indirect
|
||||
github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230916172309-ee0ee61dde2b // indirect
|
||||
github.com/waku-org/go-zerokit-rln-arm v0.0.0-20230916171929-1dd9494ff065 // indirect
|
||||
github.com/waku-org/go-zerokit-rln-x86_64 v0.0.0-20230916171518-2a77c3734dd1 // indirect
|
||||
github.com/wk8/go-ordered-map v1.0.0 // indirect
|
||||
go.uber.org/dig v1.17.1 // indirect
|
||||
go.uber.org/fx v1.20.1 // indirect
|
||||
go.uber.org/mock v0.3.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/sync v0.4.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
|
||||
golang.org/x/tools v0.14.0 // indirect
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/blake3 v1.2.1 // indirect
|
||||
)
|
||||
52
main_test.go
Normal file
52
main_test.go
Normal file
@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
)
|
||||
|
||||
// If using vscode, go to Preferences > Settings, and edit Go: Test Timeout to at least 60s
|
||||
|
||||
// List of store nodes
|
||||
var nodeList = []string{
|
||||
"/dns4/store-01.do-ams3.shards.test.statusim.net/tcp/30303/p2p/16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT",
|
||||
"/dns4/store-02.do-ams3.shards.test.statusim.net/tcp/30303/p2p/16Uiu2HAm9aDJPkhGxc2SFcEACTFdZ91Q5TJjp76qZEhq9iF59x7R",
|
||||
"/dns4/store-01.gc-us-central1-a.shards.test.statusim.net/tcp/30303/p2p/16Uiu2HAmMELCo218hncCtTvC2Dwbej3rbyHQcR8erXNnKGei7WPZ",
|
||||
"/dns4/store-02.gc-us-central1-a.shards.test.statusim.net/tcp/30303/p2p/16Uiu2HAmJnVR7ZzFaYvciPVafUXuYGLHPzSUigqAmeNw9nJUVGeM",
|
||||
"/dns4/store-01.ac-cn-hongkong-c.shards.test.statusim.net/tcp/30303/p2p/16Uiu2HAm2M7xs7cLPc3jamawkEqbr7cUJX11uvY7LxQ6WFUdUKUT",
|
||||
"/dns4/store-02.ac-cn-hongkong-c.shards.test.statusim.net/tcp/30303/p2p/16Uiu2HAm9CQhsuwPR54q27kNj9iaQVfyRzTGKrhFmr94oD8ujU6P",
|
||||
}
|
||||
|
||||
var clusterID uint16 = 16 // shards.test = 16, status.prod = 0
|
||||
|
||||
// CRITERIA --------------------------------------------------------------------------
|
||||
var pubsubTopic = "/waku/2/rs/16/32"
|
||||
var contentTopics = []string{} // "/waku/1/0xaabbccdd/rfc26"
|
||||
var startTime = time.Now().Add(-1 * time.Minute) // time.Unix(0, 1705486902684656000).Add(-60 * time.Second)
|
||||
var endTime = time.Now() // time.Unix(0, 1705486902684656000).Add(60 * time.Second)
|
||||
var envelopeHash = "0x" // Use "0x" to find all messages that match the pubsub topic, content topic and start/end time
|
||||
|
||||
func (s *StoreSuite) TestFindMessage() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
addNodes(ctx, s.node)
|
||||
hash, err := hexutil.Decode(envelopeHash)
|
||||
if err != nil {
|
||||
panic("invalid envelope hash id")
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
for _, addr := range nodeList {
|
||||
wg.Add(1)
|
||||
func(addr string) {
|
||||
defer wg.Done()
|
||||
_, err := queryNode(ctx, s.node, addr, pubsubTopic, contentTopics, startTime, endTime, hash)
|
||||
s.NoError(err)
|
||||
}(addr)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
37
setup_test.go
Normal file
37
setup_test.go
Normal file
@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
"github.com/waku-org/go-waku/waku/v2/node"
|
||||
)
|
||||
|
||||
func TestStoreSuite(t *testing.T) {
|
||||
suite.Run(t, new(StoreSuite))
|
||||
}
|
||||
|
||||
type StoreSuite struct {
|
||||
suite.Suite
|
||||
node *node.WakuNode
|
||||
}
|
||||
|
||||
func (s *StoreSuite) SetupSuite() {
|
||||
wakuNode, err := node.New(
|
||||
node.WithNTP(),
|
||||
node.WithWakuRelayAndMinPeers(1),
|
||||
node.WithClusterID(clusterID),
|
||||
)
|
||||
|
||||
s.NoError(err)
|
||||
|
||||
err = wakuNode.Start(context.Background())
|
||||
s.NoError(err)
|
||||
|
||||
s.node = wakuNode
|
||||
}
|
||||
|
||||
func (s *StoreSuite) TearDownSuite() {
|
||||
s.node.Stop()
|
||||
}
|
||||
93
utils_test.go
Normal file
93
utils_test.go
Normal file
@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
"github.com/waku-org/go-waku/logging"
|
||||
"github.com/waku-org/go-waku/waku/v2/node"
|
||||
"github.com/waku-org/go-waku/waku/v2/peerstore"
|
||||
"github.com/waku-org/go-waku/waku/v2/protocol/store"
|
||||
"github.com/waku-org/go-waku/waku/v2/utils"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var log = utils.Logger().Named("message-finder")
|
||||
|
||||
func addNodes(ctx context.Context, node *node.WakuNode) {
|
||||
for _, addr := range nodeList {
|
||||
ma, err := multiaddr.NewMultiaddr(addr)
|
||||
if err != nil {
|
||||
log.Error("invalid multiaddress", zap.Error(err), zap.String("addr", addr))
|
||||
continue
|
||||
}
|
||||
|
||||
_ = ma
|
||||
|
||||
_, err = node.AddPeer(ma, peerstore.Static, []string{string(store.StoreID_v20beta4)})
|
||||
if err != nil {
|
||||
log.Error("could not add peer", zap.Error(err), zap.Stringer("addr", ma))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func queryNode(ctx context.Context, node *node.WakuNode, addr string, pubsubTopic string, contentTopics []string, startTime time.Time, endTime time.Time, envelopeHash []byte) (int, error) {
|
||||
p, err := multiaddr.NewMultiaddr(addr)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
info, err := peer.AddrInfoFromP2pAddr(p)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
cnt := 0
|
||||
cursorIterations := 0
|
||||
|
||||
result, err := node.Store().Query(ctx, store.Query{
|
||||
PubsubTopic: pubsubTopic,
|
||||
ContentTopics: contentTopics,
|
||||
StartTime: proto.Int64(startTime.UnixNano()),
|
||||
EndTime: proto.Int64(endTime.UnixNano()),
|
||||
}, store.WithPeer(info.ID), store.WithPaging(false, 20), store.WithRequestID([]byte{1, 2, 3, 4, 5, 6, 7, 8}))
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
for {
|
||||
hasNext, err := result.Next(ctx)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
if !hasNext { // No more messages available
|
||||
break
|
||||
}
|
||||
cursorIterations += 1
|
||||
|
||||
// uncomment to find message by ID
|
||||
for _, m := range result.GetMessages() {
|
||||
if len(envelopeHash) != 0 && bytes.Equal(m.Hash(pubsubTopic), envelopeHash) {
|
||||
log.Info("⚠️⚠️⚠️ MESSAGE FOUND ⚠️⚠️⚠️", logging.HexBytes("envelopeHash", envelopeHash), logging.HostID("storeNode", info.ID))
|
||||
return 0, nil
|
||||
} else {
|
||||
log.Info(hex.EncodeToString(m.Hash(pubsubTopic)), zap.String("contentTopic", m.ContentTopic), zap.String("timestamp", fmt.Sprintf("%d", m.GetTimestamp())), logging.HostID("storeNode", info.ID), zap.Int("page", cursorIterations))
|
||||
}
|
||||
}
|
||||
|
||||
cnt += len(result.GetMessages())
|
||||
}
|
||||
|
||||
log.Info(fmt.Sprintf("%d messages found in %s (Used cursor %d times)\n", cnt, info.ID, cursorIterations))
|
||||
|
||||
return cnt, nil
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user