go-waku/cmd/waku/node.go

package main

import (
	"context"
	"crypto/ecdsa"
	"database/sql"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
	"github.com/pbnjay/memory"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/urfave/cli/v2"

	dbutils "github.com/waku-org/go-waku/waku/persistence/utils"
	wakupeerstore "github.com/waku-org/go-waku/waku/v2/peerstore"
	"github.com/waku-org/go-waku/waku/v2/rendezvous"

	"github.com/ethereum/go-ethereum/accounts/keystore"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/p2p/enode"
	dssql "github.com/ipfs/go-ds-sql"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"

	"github.com/libp2p/go-libp2p"
	"github.com/libp2p/go-libp2p/config"
	"github.com/libp2p/go-libp2p/core/peer"
	"github.com/libp2p/go-libp2p/core/protocol"
	"github.com/libp2p/go-libp2p/p2p/transport/tcp"

	pubsub "github.com/libp2p/go-libp2p-pubsub"
	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoreds"
	ws "github.com/libp2p/go-libp2p/p2p/transport/websocket"
	"github.com/multiformats/go-multiaddr"
	"github.com/waku-org/go-waku/cmd/waku/server/rest"
	"github.com/waku-org/go-waku/cmd/waku/server/rpc"
	"github.com/waku-org/go-waku/logging"
	"github.com/waku-org/go-waku/waku/metrics"
	"github.com/waku-org/go-waku/waku/persistence"
	"github.com/waku-org/go-waku/waku/v2/dnsdisc"
	"github.com/waku-org/go-waku/waku/v2/node"
	wprotocol "github.com/waku-org/go-waku/waku/v2/protocol"
	"github.com/waku-org/go-waku/waku/v2/protocol/filter"
	"github.com/waku-org/go-waku/waku/v2/protocol/legacy_filter"
	"github.com/waku-org/go-waku/waku/v2/protocol/lightpush"
	"github.com/waku-org/go-waku/waku/v2/protocol/peer_exchange"
	"github.com/waku-org/go-waku/waku/v2/protocol/relay"
	"github.com/waku-org/go-waku/waku/v2/protocol/store"
	"github.com/waku-org/go-waku/waku/v2/utils"
)

func requiresDB(options NodeOptions) bool {
	return options.Store.Enable || options.Rendezvous.Enable
}

func scalePerc(value float64) float64 {
	if value > 100 {
		return 100
	}

	if value < 0.1 {
		return 0.1
	}

	return value
}

const dialTimeout = 7 * time.Second

func nonRecoverErrorMsg(format string, a ...any) error {
	err := fmt.Errorf(format, a...)
	return nonRecoverError(err)
}

func nonRecoverError(err error) error {
	return cli.Exit(err.Error(), 166)
}

// Execute starts a go-waku node with settings determined by the Options parameter
func Execute(options NodeOptions) error {
	// Set encoding for logs (console, json, ...)
	// Note that libp2p reads the encoding from GOLOG_LOG_FMT env var.
	utils.InitLogger(options.LogEncoding, options.LogOutput)

	hostAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", options.Address, options.Port))
	if err != nil {
		return nonRecoverErrorMsg("invalid host address: %w", err)
	}

	prvKey, err := getPrivKey(options)
	if err != nil {
		return err
	}

	p2pPrvKey := utils.EcdsaPrivKeyToSecp256k1PrivKey(prvKey)
	id, err := peer.IDFromPublicKey(p2pPrvKey.GetPublic())
	if err != nil {
		return err
	}

	logger := utils.Logger().With(logging.HostID("node", id))

	var db *sql.DB
	var migrationFn func(*sql.DB) error
	if requiresDB(options) && options.Store.Migration {
		dbSettings := dbutils.DBSettings{
			Vacuum: options.Store.Vacuum,
		}
		db, migrationFn, err = dbutils.ExtractDBAndMigration(options.Store.DatabaseURL, dbSettings, logger)
		if err != nil {
			return nonRecoverErrorMsg("could not connect to DB: %w", err)
		}
	}

	ctx := context.Background()

	var metricsServer *metrics.Server
	if options.Metrics.Enable {
		metricsServer = metrics.NewMetricsServer(options.Metrics.Address, options.Metrics.Port, logger)
		go metricsServer.Start()
	}

	lvl, err := zapcore.ParseLevel(options.LogLevel)
	if err != nil {
		return err
	}

	nodeOpts := []node.WakuNodeOption{
		node.WithLogger(logger),
		node.WithLogLevel(lvl),
		node.WithPrivateKey(prvKey),
		node.WithHostAddress(hostAddr),
		node.WithKeepAlive(options.KeepAlive),
		node.WithMaxPeerConnections(options.MaxPeerConnections),
		node.WithPrometheusRegisterer(prometheus.DefaultRegisterer),
		node.WithPeerStoreCapacity(options.PeerStoreCapacity),
		node.WithClusterID(uint16(options.ClusterID)),
	}
	if len(options.AdvertiseAddresses) != 0 {
		nodeOpts = append(nodeOpts, node.WithAdvertiseAddresses(options.AdvertiseAddresses...))
	}

	if options.ExtIP != "" {
		ip := net.ParseIP(options.ExtIP)
		if ip == nil {
			return nonRecoverErrorMsg("could not set external IP address: invalid IP")
		}

		nodeOpts = append(nodeOpts, node.WithExternalIP(ip))
	}

	if options.DNS4DomainName != "" {
		nodeOpts = append(nodeOpts, node.WithDNS4Domain(options.DNS4DomainName))
	}

	libp2pOpts := node.DefaultLibP2POptions

	libp2pOpts = append(libp2pOpts, libp2p.PrometheusRegisterer(prometheus.DefaultRegisterer))

	memPerc := scalePerc(options.ResourceScalingMemoryPercent)
	fdPerc := scalePerc(options.ResourceScalingFDPercent)
	limits := rcmgr.DefaultLimits // Default memory limit: 1/8th of total memory, minimum 128MB, maximum 1GB
	scaledLimits := limits.Scale(int64(float64(memory.TotalMemory())*memPerc/100), int(float64(getNumFDs())*fdPerc/100))
	resourceManager, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(scaledLimits))
	if err != nil {
		return fmt.Errorf("could not set resource limits: %w", err)
	}

	libp2pOpts = append(libp2pOpts, libp2p.ResourceManager(resourceManager))
	libp2p.SetDefaultServiceLimits(&limits)

	if len(options.AdvertiseAddresses) == 0 {
		libp2pOpts = append(libp2pOpts, libp2p.NATPortMap()) // Attempt to open ports using uPNP for NATed hosts.)
	}

	// Node can be a circuit relay server
	if options.CircuitRelay {
		libp2pOpts = append(libp2pOpts, libp2p.EnableRelayService())
	}

	if options.ForceReachability != "" {
		libp2pOpts = append(libp2pOpts, libp2p.EnableRelay())
		nodeOpts = append(nodeOpts, node.WithCircuitRelayParams(2*time.Second, 2*time.Second))
		if options.ForceReachability == "private" {
			logger.Warn("node forced to be unreachable!")
			libp2pOpts = append(libp2pOpts, libp2p.ForceReachabilityPrivate())
		} else if options.ForceReachability == "public" {
			logger.Warn("node forced to be publicly reachable!")
			libp2pOpts = append(libp2pOpts, libp2p.ForceReachabilityPublic())
		} else {
			return nonRecoverErrorMsg("invalid reachability value")
		}
	}

	if options.UserAgent != "" {
		libp2pOpts = append(libp2pOpts, libp2p.UserAgent(options.UserAgent))
	}

	if options.Websocket.Enable {
		nodeOpts = append(nodeOpts, node.WithWebsockets(options.Websocket.Address, options.Websocket.WSPort))
	}

	if options.Websocket.Secure {
		nodeOpts = append(nodeOpts, node.WithSecureWebsockets(options.Websocket.Address, options.Websocket.WSSPort, options.Websocket.CertPath, options.Websocket.KeyPath))
	}

	if options.ShowAddresses {
		printListeningAddresses(ctx, nodeOpts, options)
		return nil
	}

	if options.Store.Enable && options.PersistPeers {
		// Create persistent peerstore
		queries, err := dbutils.NewQueries("peerstore", db)
		if err != nil {
			return nonRecoverErrorMsg("could not setup persistent peerstore database: %w", err)

		}

		datastore := dssql.NewDatastore(db, queries)
		opts := pstoreds.DefaultOpts()
		peerStore, err := pstoreds.NewPeerstore(ctx, datastore, opts)
		if err != nil {
			return nonRecoverErrorMsg("could not create persistent peerstore: %w", err)
		}

		nodeOpts = append(nodeOpts, node.WithPeerStore(peerStore))
	}

	nodeOpts = append(nodeOpts, node.WithLibP2POptions(libp2pOpts...))
	nodeOpts = append(nodeOpts, node.WithNTP())

	if options.Relay.Enable {
		var wakurelayopts []pubsub.Option
		wakurelayopts = append(wakurelayopts, pubsub.WithPeerExchange(options.Relay.PeerExchange))
		nodeOpts = append(nodeOpts, node.WithWakuRelayAndMinPeers(options.Relay.MinRelayPeersToPublish, wakurelayopts...))
	}

	nodeOpts = append(nodeOpts, node.WithWakuFilterLightNode())

	if options.Filter.Enable {
		nodeOpts = append(nodeOpts, node.WithWakuFilterFullNode(filter.WithTimeout(options.Filter.Timeout)))
	}

	if options.Filter.UseV1 {
		nodeOpts = append(nodeOpts, node.WithLegacyWakuFilter(!options.Filter.DisableFullNode, legacy_filter.WithTimeout(options.Filter.Timeout)))
	}

	var dbStore *persistence.DBStore
	if requiresDB(options) {
		dbOptions := []persistence.DBOption{
			persistence.WithDB(db),
			persistence.WithRetentionPolicy(options.Store.RetentionMaxMessages, options.Store.RetentionTime),
		}

		if options.Store.Migration {
			dbOptions = append(dbOptions, persistence.WithMigrations(migrationFn)) // TODO: refactor migrations out of DBStore, or merge DBStore with rendezvous DB
		}

		dbStore, err = persistence.NewDBStore(prometheus.DefaultRegisterer, logger, dbOptions...)
		if err != nil {
			return nonRecoverErrorMsg("error setting up db store: %w", err)
		}

		nodeOpts = append(nodeOpts, node.WithMessageProvider(dbStore))
	}

	if options.Store.Enable {
		nodeOpts = append(nodeOpts, node.WithWakuStore())
		nodeOpts = append(nodeOpts, node.WithMessageProvider(dbStore))
	}

	if options.LightPush.Enable {
		nodeOpts = append(nodeOpts, node.WithLightPush())
	}

	var discoveredNodes []dnsdisc.DiscoveredNode
	if options.DNSDiscovery.Enable {
		if len(options.DNSDiscovery.URLs.Value()) != 0 {
			for _, url := range options.DNSDiscovery.URLs.Value() {
				logger.Info("attempting DNS discovery with ", zap.String("URL", url))
				nodes, err := dnsdisc.RetrieveNodes(ctx, url, dnsdisc.WithNameserver(options.DNSDiscovery.Nameserver))
				if err != nil {
					logger.Warn("dns discovery error ", zap.Error(err))
				} else {
					var discPeerInfo []peer.AddrInfo
					for _, n := range nodes {
						discPeerInfo = append(discPeerInfo, n.PeerInfo)
					}
					logger.Info("found dns entries ", zap.Any("nodes", discPeerInfo))
					discoveredNodes = append(discoveredNodes, nodes...)
				}
			}
		} else {
			return nonRecoverErrorMsg("DNS discovery URL is required")
		}
	}

	if options.DiscV5.Enable {
		var bootnodes []*enode.Node
		for _, addr := range options.DiscV5.Nodes.Value() {
			bootnode, err := enode.Parse(enode.ValidSchemes, addr)
			if err != nil {
				logger.Fatal("parsing ENR", zap.Error(err))
			}
			bootnodes = append(bootnodes, bootnode)
		}

		for _, n := range discoveredNodes {
			if n.ENR != nil {
				bootnodes = append(bootnodes, n.ENR)
			}
		}

		nodeOpts = append(nodeOpts, node.WithDiscoveryV5(options.DiscV5.Port, bootnodes, options.DiscV5.AutoUpdate))
	}

	if options.PeerExchange.Enable {
		nodeOpts = append(nodeOpts, node.WithPeerExchange())
	}

	if options.Rendezvous.Enable {
		rdb := rendezvous.NewDB(db, logger)
		nodeOpts = append(nodeOpts, node.WithRendezvous(rdb))
	}

	utils.Logger().Info("Version details ", zap.String("version", node.Version), zap.String("commit", node.GitCommit))

	if err = checkForRLN(logger, options, &nodeOpts); err != nil {
		return nonRecoverError(err)
	}

	wakuNode, err := node.New(nodeOpts...)
	if err != nil {
		return fmt.Errorf("could not instantiate waku: %w", err)
	}

	//Process pubSub and contentTopics specified and arrive at all corresponding pubSubTopics
	pubSubTopicMap, err := processTopics(options)
	if err != nil {
		return nonRecoverError(err)
	}

	pubSubTopicMapKeys := make([]string, 0, len(pubSubTopicMap))
	for k := range pubSubTopicMap {
		pubSubTopicMapKeys = append(pubSubTopicMapKeys, k)
	}

	if options.Filter.UseV1 {
		if err := addStaticPeers(wakuNode, options.Filter.NodesV1, pubSubTopicMapKeys, legacy_filter.FilterID_v20beta1); err != nil {
			return err
		}
	}

	if err = wakuNode.Start(ctx); err != nil {
		return nonRecoverError(err)
	}

	for _, d := range discoveredNodes {
		wakuNode.AddDiscoveredPeer(d.PeerID, d.PeerInfo.Addrs, wakupeerstore.DNSDiscovery, nil, true)
	}

	//For now assuming that static peers added support/listen on all topics specified via commandLine.
	staticPeers := map[protocol.ID][]multiaddr.Multiaddr{
		store.StoreID_v20beta4:            options.Store.Nodes,
		lightpush.LightPushID_v20beta1:    options.LightPush.Nodes,
		rendezvous.RendezvousID:           options.Rendezvous.Nodes,
		filter.FilterSubscribeID_v20beta1: options.Filter.Nodes,
	}
	for protocolID, peers := range staticPeers {
		if err = addStaticPeers(wakuNode, peers, pubSubTopicMapKeys, protocolID); err != nil {
			return err
		}
	}

	var wg sync.WaitGroup

	if options.Relay.Enable {
		for nodeTopic, cTopics := range pubSubTopicMap {
			nodeTopic := nodeTopic
			_, err := wakuNode.Relay().Subscribe(ctx, wprotocol.NewContentFilter(nodeTopic, cTopics...), relay.WithoutConsumer())
			if err != nil {
				return err
			}

			if len(options.Rendezvous.Nodes) != 0 {
				// Register the node in rendezvous point
				iter := rendezvous.NewRendezvousPointIterator(options.Rendezvous.Nodes)

				wg.Add(1)
				go func(nodeTopic string) {
					t := time.NewTicker(rendezvous.RegisterDefaultTTL)
					defer t.Stop()
					defer wg.Done()

					for {
						select {
						case <-ctx.Done():
							return
						case <-t.C:
							// Register in rendezvous points periodically
							wakuNode.Rendezvous().RegisterWithNamespace(ctx, nodeTopic, iter.RendezvousPoints())
						}
					}
				}(nodeTopic)

				wg.Add(1)
				go func(nodeTopic string) {
					defer wg.Done()
					desiredOutDegree := wakuNode.Relay().Params().D
					t := time.NewTicker(7 * time.Second)
					defer t.Stop()
					for {
						select {
						case <-ctx.Done():
							return
						case <-t.C:
							peerCnt := len(wakuNode.Relay().PubSub().ListPeers(nodeTopic))
							peersToFind := desiredOutDegree - peerCnt
							if peersToFind <= 0 {
								continue
							}

							rp := <-iter.Next(ctx)
							if rp == nil {
								continue
							}
							ctx, cancel := context.WithTimeout(ctx, 7*time.Second)
							wakuNode.Rendezvous().DiscoverWithNamespace(ctx, nodeTopic, rp, peersToFind)
							cancel()
						}
					}
				}(nodeTopic)

			}
		}

		for _, protectedTopic := range options.Relay.ProtectedTopics {
			if err := wakuNode.Relay().AddSignedTopicValidator(protectedTopic.Topic, protectedTopic.PublicKey); err != nil {
				return nonRecoverErrorMsg("could not add signed topic validator: %w", err)
			}
		}
	}

	for _, n := range options.StaticNodes {
		go func(ctx context.Context, node multiaddr.Multiaddr) {
			ctx, cancel := context.WithTimeout(ctx, dialTimeout)
			defer cancel()
			err = wakuNode.DialPeerWithMultiAddress(ctx, node)
			if err != nil {
				logger.Error("dialing peer", zap.Error(err))
			}
		}(ctx, n)
	}

	if options.DiscV5.Enable {
		if err = wakuNode.DiscV5().Start(ctx); err != nil {
			logger.Fatal("starting discovery v5", zap.Error(err))
		}
	}

	// retrieve and connect to peer exchange peers
	if options.PeerExchange.Enable && options.PeerExchange.Node != nil {
		logger.Info("retrieving peer info via peer exchange protocol")

		peerID, err := wakuNode.AddPeer(*options.PeerExchange.Node, wakupeerstore.Static,
			pubSubTopicMapKeys, peer_exchange.PeerExchangeID_v20alpha1)
		if err != nil {
			logger.Error("adding peer exchange peer", logging.MultiAddrs("node", *options.PeerExchange.Node), zap.Error(err))
		} else {
			desiredOutDegree := wakuNode.Relay().Params().D
			if err = wakuNode.PeerExchange().Request(ctx, desiredOutDegree, peer_exchange.WithPeer(peerID)); err != nil {
				logger.Error("requesting peers via peer exchange", zap.Error(err))
			}
		}
	}

	if len(discoveredNodes) != 0 {
		for _, n := range discoveredNodes {
			go func(ctx context.Context, info peer.AddrInfo) {
				ctx, cancel := context.WithTimeout(ctx, dialTimeout)
				defer cancel()
				err = wakuNode.DialPeerWithInfo(ctx, info)
				if err != nil {
					logger.Error("dialing peer", logging.HostID("peer", info.ID), zap.Error(err))
				}
			}(ctx, n.PeerInfo)

		}
	}

	var rpcServer *rpc.WakuRPC
	if options.RPCServer.Enable {
		rpcServer = rpc.NewWakuRPC(wakuNode, options.RPCServer.Address, options.RPCServer.Port, options.RPCServer.Admin, options.PProf, options.RPCServer.RelayCacheCapacity, logger)
		rpcServer.Start()
	}

	var restServer *rest.WakuRest
	if options.RESTServer.Enable {
		wg.Add(1)
		restServer = rest.NewWakuRest(wakuNode, options.RESTServer.Address, options.RESTServer.Port, options.PProf, options.RESTServer.RelayCacheCapacity, logger)
		restServer.Start(ctx, &wg)
	}

	wg.Wait()
	logger.Info("Node setup complete")

	// Wait for a SIGINT or SIGTERM signal
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	<-ch
	logger.Info("Received signal, shutting down...")

	// shut the node down
	wakuNode.Stop()

	if options.RPCServer.Enable {
		if err := rpcServer.Stop(ctx); err != nil {
			return err
		}
	}

	if options.RESTServer.Enable {
		if err := restServer.Stop(ctx); err != nil {
			return err
		}
	}

	if options.Metrics.Enable {
		if err = metricsServer.Stop(ctx); err != nil {
			return err
		}
	}

	if db != nil {
		if err = db.Close(); err != nil {
			return err
		}
	}

	return nil
}

func processTopics(options NodeOptions) (map[string][]string, error) {

	//Using a map to avoid duplicate pub-sub topics that can result from autosharding
	// or same-topic being passed twice.
	pubSubTopicMap := make(map[string][]string)

	for _, topic := range options.Relay.Topics.Value() {
		pubSubTopicMap[topic] = []string{}
	}

	for _, topic := range options.Relay.PubSubTopics.Value() {
		pubSubTopicMap[topic] = []string{}
	}

	//Get pubSub topics from contentTopics if they are as per autosharding
	for _, cTopic := range options.Relay.ContentTopics.Value() {
		contentTopic, err := wprotocol.StringToContentTopic(cTopic)
		if err != nil {
			return nil, err
		}
		pTopic := wprotocol.GetShardFromContentTopic(contentTopic, wprotocol.GenerationZeroShardsCount)
		if _, ok := pubSubTopicMap[pTopic.String()]; !ok {
			pubSubTopicMap[pTopic.String()] = []string{}
		}
		pubSubTopicMap[pTopic.String()] = append(pubSubTopicMap[pTopic.String()], cTopic)
	}
	//If no topics are passed, then use default waku topic.
	if len(pubSubTopicMap) == 0 {
		pubSubTopicMap[relay.DefaultWakuTopic] = []string{}
	}

	return pubSubTopicMap, nil
}

func addStaticPeers(wakuNode *node.WakuNode, addresses []multiaddr.Multiaddr, pubSubTopics []string, protocols ...protocol.ID) error {
	for _, addr := range addresses {
		_, err := wakuNode.AddPeer(addr, wakupeerstore.Static, pubSubTopics, protocols...)
		if err != nil {
			return fmt.Errorf("could not add static peer: %w", err)
		}
	}
	return nil
}

func loadPrivateKeyFromFile(path string, passwd string) (*ecdsa.PrivateKey, error) {
	src, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var encryptedK keystore.CryptoJSON
	err = json.Unmarshal(src, &encryptedK)
	if err != nil {
		return nil, err
	}

	pKey, err := keystore.DecryptDataV3(encryptedK, passwd)
	if err != nil {
		return nil, err
	}

	return crypto.ToECDSA(pKey)
}

func getPrivKey(options NodeOptions) (*ecdsa.PrivateKey, error) {
	var prvKey *ecdsa.PrivateKey
	// get private key from nodeKey or keyFile
	if options.NodeKey != nil {
		prvKey = options.NodeKey
	} else {
		if _, err := os.Stat(options.KeyFile); err == nil {
			if prvKey, err = loadPrivateKeyFromFile(options.KeyFile, options.KeyPasswd); err != nil {
				return nil, fmt.Errorf("could not read keyfile: %w", err)
			}
		} else {
			if os.IsNotExist(err) {
				if prvKey, err = crypto.GenerateKey(); err != nil {
					return nil, fmt.Errorf("error generating key: %w", err)
				}
			} else {
				return nil, fmt.Errorf("could not read keyfile: %w", err)
			}
		}
	}
	return prvKey, nil
}

func printListeningAddresses(ctx context.Context, nodeOpts []node.WakuNodeOption, options NodeOptions) {
	params := new(node.WakuNodeParameters)
	for _, opt := range nodeOpts {
		err := opt(params)
		if err != nil {
			panic(err)
		}
	}

	var libp2pOpts []config.Option
	libp2pOpts = append(libp2pOpts,
		params.Identity(),
		libp2p.ListenAddrs(params.MultiAddresses()...),
	)

	if options.Websocket.Secure {
		transports := libp2p.ChainOptions(
			libp2p.Transport(tcp.NewTCPTransport),
			libp2p.Transport(ws.New, ws.WithTLSConfig(params.TLSConfig())),
		)
		libp2pOpts = append(libp2pOpts, transports)
	}

	addrFactory := params.AddressFactory()
	if addrFactory != nil {
		libp2pOpts = append(libp2pOpts, libp2p.AddrsFactory(addrFactory))
	}

	h, err := libp2p.New(libp2pOpts...)
	if err != nil {
		panic(err)
	}

	hostAddrs := utils.EncapsulatePeerID(h.ID(), h.Addrs()...)
	for _, addr := range hostAddrs {
		fmt.Println(addr)
	}

}