Add internal RPC server and statusd-cli client (#463)

.gitignore

@@ -7,6 +7,7 @@
 /statusd-data
 /cmd/statusd/statusd-data
 /cmd/statusd/statusd
+/cmd/statusd-cli/statusd-cli
 
 /wnode-status-data
 /cmd/wnode-status/wnode-status-data

cmd/statusd-cli/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+const (
+	// Addr is the default statusd address to connect to.
+	Addr = "localhost:51515"
+)
+
+var (
+	addr = flag.String("addr", Addr, "set statusd address (default localhost:51515)")
+)
+
+// main is the entrypoint for the statusd command line interface.
+func main() {
+	flag.Usage = printUsage
+	flag.Parse()
+
+	fmt.Printf("statusd-cli connecting statusd on '%s'\n", *addr)
+
+	// Running REPL.
+	repl := NewREPL(*addr)
+	err := repl.Run()
+	if err != nil {
+		fmt.Printf("stopped with error: %v\n", err)
+		os.Exit(-1)
+	}
+}
+
+// printUsage prints a little help for statusd-cli.
+func printUsage() {
+	fmt.Fprintln(os.Stderr, "Usage: statusd-cli [options]")
+	fmt.Fprintf(os.Stderr, `
+Examples:
+  statusd-cli -addr=<address> # connect statusd on <address>
+  
+Options:
+`)
+	flag.PrintDefaults()
+}

cmd/statusd-cli/repl.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// REPL implements the read-eval-print loop for the commands
+// to be sent to statusd.
+type REPL struct {
+	addr string
+}
+
+// NewREPL creates a REPL instance communicating with the
+// addressed statusd.
+func NewREPL(addr string) *REPL {
+	return &REPL{
+		addr: addr,
+	}
+}
+
+// Run operates the loop to read a command and its arguments,
+// execute it via the client, and print the result.
+func (r *REPL) Run() error {
+	var (
+		conn   net.Conn
+		reader *bufio.Reader
+		writer *bufio.Writer
+		err    error
+	)
+	input := bufio.NewReader(os.Stdin)
+	connect := true
+	for {
+		// Connect first time and after connection errors.
+		if connect {
+			conn, err = net.Dial("tcp", r.addr)
+			if err != nil {
+				return fmt.Errorf("error connecting to statusd: %v", err)
+			}
+			connect = false
+			reader = bufio.NewReader(conn)
+			writer = bufio.NewWriter(conn)
+		}
+		// Read command line.
+		fmt.Print(">>> ")
+		command, err := input.ReadString('\n')
+		if err != nil {
+			fmt.Printf("ERR %v\n", err)
+			continue
+		}
+		// Check for possible end.
+		if strings.ToLower(command) == "quit\n" {
+			return nil
+		}
+		// Execute on statusd.
+		_, err = writer.WriteString(command)
+		if err != nil {
+			fmt.Printf("ERR %v\n", err)
+			connect = true
+			continue
+		}
+		err = writer.Flush()
+		if err != nil {
+			fmt.Printf("ERR %v\n", err)
+			connect = true
+			continue
+		}
+		// Read number of expected result lines.
+		countStr, err := reader.ReadString('\n')
+		if err != nil {
+			fmt.Printf("ERR %v\n", err)
+			connect = true
+			continue
+		}
+		count, err := strconv.Atoi(strings.TrimSuffix(countStr, "\n"))
+		if err != nil {
+			fmt.Printf("ERR %v\n", err)
+			continue
+		}
+		// Read and print result lines.
+		for i := 0; i < count; i++ {
+			reply, err := reader.ReadString('\n')
+			if err != nil {
+				fmt.Printf("ERR %v\n", err)
+				continue
+			}
+			fmt.Print("<<< ")
+			fmt.Print(reply)
+		}
+	}
+}

cmd/statusd/debug/commands.go

@@ -0,0 +1,179 @@
+package debug
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/status-im/status-go/geth/api"
+	"github.com/status-im/status-go/geth/common"
+	"github.com/status-im/status-go/geth/params"
+)
+
+// command contains the result of a parsed command line and
+// is able to execute it on the command set.
+type command struct {
+	funcName string
+	args     []interface{}
+}
+
+// newCommand parses a command line and returns the command
+// for further usage.
+func newCommand(commandLine string) (*command, error) {
+	expr, err := parser.ParseExpr(commandLine)
+	if err != nil {
+		return nil, err
+	}
+	switch expr := expr.(type) {
+	case *ast.CallExpr:
+		f, ok := expr.Fun.(*ast.Ident)
+		if !ok {
+			return nil, fmt.Errorf("invalid expression: %q", commandLine)
+		}
+		return &command{
+			funcName: f.Name,
+			args:     exprsToArgs(expr.Args),
+		}, nil
+	default:
+		return nil, fmt.Errorf("invalid command line: %q", commandLine)
+	}
+}
+
+// execute calls the method on the passed command set value.
+func (c *command) execute(commandSetValue reflect.Value) (replies []string, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("invalid API call: %v", r)
+		}
+	}()
+	method := commandSetValue.MethodByName(c.funcName)
+	if !method.IsValid() {
+		return nil, fmt.Errorf("command %q not found", c.funcName)
+	}
+	argsV := make([]reflect.Value, len(c.args))
+	for i, arg := range c.args {
+		argsV[i] = reflect.ValueOf(arg)
+	}
+	repliesV := method.Call(argsV)
+	replies = make([]string, len(repliesV))
+	for i, replyV := range repliesV {
+		replies[i] = fmt.Sprintf("%v", replyV)
+	}
+	return replies, nil
+}
+
+// exprsToArgs converts the argument expressions to arguments.
+func exprsToArgs(exprs []ast.Expr) []interface{} {
+	args := make([]interface{}, len(exprs))
+	for i, expr := range exprs {
+		switch expr := expr.(type) {
+		case *ast.BasicLit:
+			switch expr.Kind {
+			case token.INT:
+				args[i], _ = strconv.ParseInt(expr.Value, 10, 64)
+			case token.FLOAT:
+				args[i], _ = strconv.ParseFloat(expr.Value, 64)
+			case token.CHAR:
+				args[i] = expr.Value[1]
+			case token.STRING:
+				r := strings.NewReplacer("\\n", "\n", "\\t", "\t", "\\\"", "\"")
+				args[i] = strings.Trim(r.Replace(expr.Value), `"`)
+			}
+		case *ast.Ident:
+			switch expr.Name {
+			case "true":
+				args[i] = true
+			case "false":
+				args[i] = false
+			default:
+				args[i] = expr.Name
+			}
+		default:
+			args[i] = fmt.Sprintf("[unknown: %#v]", expr)
+		}
+	}
+	return args
+}
+
+// commandSet implements the set of commands the debugger unterstands.
+// In the beginning a subset of the Status API, may later grow to
+// utility commands.
+//
+// Direct invocation of commands on the Status API sometimes sadly
+// is not possible due to non-low-level arguments. Here this wrapper
+// helps.
+type commandSet struct {
+	statusAPI *api.StatusAPI
+}
+
+// newCommandSet creates the command set for the passed Status API
+// instance.
+func newCommandSet(statusAPI *api.StatusAPI) *commandSet {
+	return &commandSet{
+		statusAPI: statusAPI,
+	}
+}
+
+// StartNode loads the configuration out of the passed string and
+// starts a node with it.
+func (cs *commandSet) StartNode(config string) error {
+	nodeConfig, err := params.LoadNodeConfig(config)
+	if err != nil {
+		return err
+	}
+	return cs.statusAPI.StartNode(nodeConfig)
+}
+
+// StopNode starts the stopped node.
+func (cs *commandSet) StopNode() error {
+	return cs.statusAPI.StopNode()
+}
+
+// ResetChainData removes chain data from data directory.
+func (cs *commandSet) ResetChainData() error {
+	_, err := cs.statusAPI.ResetChainDataAsync()
+	return err
+}
+
+// CallRPC calls status node via RPC.
+func (cs *commandSet) CallRPC(inputJSON string) string {
+	return cs.statusAPI.CallRPC(inputJSON)
+}
+
+// CreateAccount creates an internal geth account.
+func (cs *commandSet) CreateAccount(password string) (string, string, string, error) {
+	return cs.statusAPI.CreateAccount(password)
+}
+
+// CreateChildAccount creates a sub-account.
+func (cs *commandSet) CreateChildAccount(parentAddress, password string) (string, string, error) {
+	return cs.statusAPI.CreateChildAccount(parentAddress, password)
+}
+
+// RecoverAccount re-creates the master key using the given details.
+func (cs *commandSet) RecoverAccount(password, mnemonic string) (string, string, error) {
+	return cs.statusAPI.RecoverAccount(password, mnemonic)
+}
+
+// SelectAccount selects the addressed account.
+func (cs *commandSet) SelectAccount(address, password string) error {
+	return cs.statusAPI.SelectAccount(address, password)
+}
+
+// Logout clears the Whisper identities.
+func (cs *commandSet) Logout() error {
+	return cs.statusAPI.Logout()
+}
+
+// CompleteTransaction instructs API to complete sending of a given transaction.
+func (cs *commandSet) CompleteTransaction(id, password string) (string, error) {
+	txHash, err := cs.statusAPI.CompleteTransaction(common.QueuedTxID(id), password)
+	if err != nil {
+		return "", err
+	}
+	return txHash.String(), nil
+}

cmd/statusd/debug/debug.go

@@ -0,0 +1,110 @@
+package debug
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"net"
+	"reflect"
+
+	"github.com/status-im/status-go/geth/api"
+)
+
+const (
+	// CLIPort is the CLI port.
+	CLIPort = "51515"
+)
+
+// Server provides a debug server receiving line based commands from
+// a CLI via the debugging port and executing those on the Status API
+// using reflection. The returned values will be rendered as
+// string and returned to the CLI.
+type Server struct {
+	commandSetValue reflect.Value
+	listener        net.Listener
+}
+
+// New creates a debug server using the passed Status API.
+// It also starts the server.
+func New(statusAPI *api.StatusAPI, port string) (*Server, error) {
+	listener, err := net.Listen("tcp", fmt.Sprintf(":%s", port)) // nolint
+	if err != nil {
+		return nil, err
+	}
+	s := Server{
+		commandSetValue: reflect.ValueOf(newCommandSet(statusAPI)),
+		listener:        listener,
+	}
+	go s.backend()
+	return &s, nil
+}
+
+// backend receives the commands and executes them on
+// the Status API.
+func (s *Server) backend() {
+	for {
+		conn, err := s.listener.Accept()
+		if err != nil {
+			log.Printf("cannot establish debug connection: %v", err)
+			continue
+		}
+		go s.handleConnection(conn)
+	}
+}
+
+// handleConnection handles all commands of one connection.
+func (s *Server) handleConnection(conn net.Conn) {
+	reader := bufio.NewReader(conn)
+	writer := bufio.NewWriter(conn)
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Printf("error while closing debug connection: %v", err)
+		}
+	}()
+	// Read, execute, and respond commands of a session.
+	for {
+		var (
+			replies []string
+			err     error
+		)
+		command, err := s.readCommandLine(reader)
+		if err != nil {
+			replies = []string{fmt.Sprintf("cannot read command: %v", err)}
+		} else {
+			replies, err = command.execute(s.commandSetValue)
+			if err != nil {
+				replies = []string{fmt.Sprintf("cannot execute command: %v", err)}
+			}
+		}
+		err = s.writeReplies(writer, replies)
+		if err != nil {
+			log.Printf("cannot write replies: %v", err)
+			return
+		}
+	}
+}
+
+// readCommandLine receives a command line via network and
+// parses it into an executable command.
+func (s *Server) readCommandLine(reader *bufio.Reader) (*command, error) {
+	commandLine, err := reader.ReadString('\n')
+	if err != nil {
+		return nil, err
+	}
+	return newCommand(commandLine)
+}
+
+// writeReplies sends the replies back to the CLI.
+func (s *Server) writeReplies(writer *bufio.Writer, replies []string) error {
+	_, err := fmt.Fprintf(writer, "%d\n", len(replies))
+	if err != nil {
+		return err
+	}
+	for i, reply := range replies {
+		_, err = fmt.Fprintf(writer, "[%d] %s\n", i, reply)
+		if err != nil {
+			return err
+		}
+	}
+	return writer.Flush()
+}

cmd/statusd/debug/debug_test.go

@@ -0,0 +1,216 @@
+package debug_test
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/status-im/status-go/cmd/statusd/debug"
+	"github.com/status-im/status-go/geth/api"
+	"github.com/status-im/status-go/geth/params"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestInvalidExpressions tests invalid expressions.
+func TestInvalidExpressions(t *testing.T) {
+	assert := assert.New(t)
+
+	startDebugging(assert)
+
+	conn := connectDebug(assert)
+	tests := []struct {
+		commandLine string
+		replies     []string
+	}{
+		{
+			commandLine: "",
+			replies:     []string{"[0] cannot read command: 1:2: expected operand, found 'EOF'"},
+		}, {
+			commandLine: "1 + 1",
+			replies:     []string{"[0] cannot read command: invalid command line: \"1 + 1\\n\""},
+		}, {
+			commandLine: "func() { panic(42) }",
+			replies:     []string{"[0] cannot read command: invalid command line: \"func() { panic(42) }\\n\""},
+		}, {
+			commandLine: "DoesNotExist()",
+			replies:     []string{"[0] cannot execute command: command \"DoesNotExist\" not found"},
+		}, {
+			commandLine: "node.Start()",
+			replies:     []string{"[0] cannot read command: invalid expression: \"node.Start()\\n\""},
+		},
+	}
+
+	for _, test := range tests {
+		replies := sendCommandLine(assert, conn, test.commandLine)
+		assert.Equal(test.replies, replies)
+	}
+}
+
+// TestStartStopNode tests starting and stopping a node remotely.
+func TestStartStopNode(t *testing.T) {
+	assert := assert.New(t)
+	configJSON, cleanup, err := mkConfigJSON("start-stop-node")
+	assert.NoError(err)
+	defer cleanup()
+
+	startDebugging(assert)
+
+	conn := connectDebug(assert)
+
+	commandLine := fmt.Sprintf("StartNode(%q)", configJSON)
+	replies := sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+
+	commandLine = "StopNode()"
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+}
+
+// TestCreateAccount tests creating an account on the server.
+func TestCreateAccount(t *testing.T) {
+	assert := assert.New(t)
+	configJSON, cleanup, err := mkConfigJSON("create-account")
+	assert.NoError(err)
+	defer cleanup()
+
+	startDebugging(assert)
+
+	conn := connectDebug(assert)
+
+	commandLine := fmt.Sprintf("StartNode(%q)", configJSON)
+	replies := sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+
+	commandLine = fmt.Sprintf("CreateAccount(%q)", "password")
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 4)
+	assert.NotEqual("[0] <nil>", replies[0])
+	assert.NotEqual("[1] <nil>", replies[1])
+	assert.NotEqual("[2] <nil>", replies[2])
+	assert.Equal("[3] <nil>", replies[3])
+
+	commandLine = "StopNode()"
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+}
+
+// TestSelectAccountLogout tests selecting an account on the server
+// and logging out afterwards.
+func TestSelectAccountLogout(t *testing.T) {
+	assert := assert.New(t)
+	configJSON, cleanup, err := mkConfigJSON("select-account")
+	assert.NoError(err)
+	defer cleanup()
+
+	startDebugging(assert)
+
+	conn := connectDebug(assert)
+
+	commandLine := fmt.Sprintf("StartNode(%q)", configJSON)
+	replies := sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+
+	commandLine = fmt.Sprintf("CreateAccount(%q)", "password")
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 4)
+	assert.NotEqual("[0] <nil>", replies[0])
+	assert.NotEqual("[1] <nil>", replies[1])
+	assert.NotEqual("[2] <nil>", replies[2])
+	assert.Equal("[3] <nil>", replies[3])
+
+	address := replies[0][4:]
+
+	commandLine = fmt.Sprintf("SelectAccount(%q, %q)", address, "password")
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+
+	commandLine = "Logout()"
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+
+	commandLine = "StopNode()"
+	replies = sendCommandLine(assert, conn, commandLine)
+	assert.Len(replies, 1)
+	assert.Equal("[0] <nil>", replies[0])
+}
+
+//-----
+// HELPERS
+//-----
+
+var (
+	mu sync.Mutex
+	d  *debug.Server
+)
+
+// startDebugging lazily creates or reuses a debug instance.
+func startDebugging(assert *assert.Assertions) {
+	mu.Lock()
+	defer mu.Unlock()
+	if d == nil {
+		var err error
+		api := api.NewStatusAPI()
+		d, err = debug.New(api, debug.CLIPort)
+		assert.NoError(err)
+	}
+}
+
+// mkConfigJSON creates a configuration matching to
+// a temporary directory and a cleanup for that directory.
+func mkConfigJSON(name string) (string, func(), error) {
+	tmpDir, err := ioutil.TempDir(os.TempDir(), name)
+	if err != nil {
+		return "", nil, err
+	}
+	cleanup := func() {
+		os.RemoveAll(tmpDir) //nolint: errcheck
+	}
+	configJSON := `{
+		"NetworkId": ` + strconv.Itoa(params.RopstenNetworkID) + `,
+		"DataDir": "` + tmpDir + `",
+		"LogLevel": "INFO",
+		"RPCEnabled": true
+	}`
+	return configJSON, cleanup, nil
+}
+
+// connectDebug connects to the debug instance.
+func connectDebug(assert *assert.Assertions) net.Conn {
+	conn, err := net.Dial("tcp", ":51515")
+	assert.NoError(err)
+	return conn
+}
+
+// sendCommandLine sends a command line via the passed connection.
+func sendCommandLine(assert *assert.Assertions, conn net.Conn, commandLine string) []string {
+	reader := bufio.NewReader(conn)
+	writer := bufio.NewWriter(conn)
+	_, err := writer.WriteString(commandLine + "\n")
+	assert.NoError(err)
+	err = writer.Flush()
+	assert.NoError(err)
+	countStr, err := reader.ReadString('\n')
+	assert.NoError(err)
+	count, err := strconv.Atoi(strings.TrimSuffix(countStr, "\n"))
+	assert.NoError(err)
+	replies := make([]string, count)
+	for i := 0; i < count; i++ {
+		reply, err := reader.ReadString('\n')
+		assert.NoError(err)
+		replies[i] = strings.TrimSuffix(reply, "\n")
+	}
+	return replies
+}

cmd/statusd/main.go

@@ -8,6 +8,7 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/status-im/status-go/cmd/statusd/debug"
 	"github.com/status-im/status-go/geth/api"
 	"github.com/status-im/status-go/geth/params"
 )
@@ -27,6 +28,8 @@ var (
 	httpEnabled    = flag.Bool("http", false, "HTTP RPC endpoint enabled (default: false)")
 	httpPort       = flag.Int("httpport", params.HTTPPort, "HTTP RPC server's listening port")
 	ipcEnabled     = flag.Bool("ipc", false, "IPC RPC endpoint enabled")
+	cliEnabled     = flag.Bool("cli", false, "Enable debugging CLI server")
+	cliPort        = flag.String("cliport", debug.CLIPort, "CLI server's listening port")
 	logLevel       = flag.String("log", "INFO", `Log level, one of: "ERROR", "WARN", "INFO", "DEBUG", and "TRACE"`)
 	logFile        = flag.String("logfile", "", "Path to the log file")
 	version        = flag.Bool("version", false, "Print version")
@@ -57,6 +60,15 @@ func main() {
 	// wait till node is started
 	<-started
 
+	// Check if debugging CLI connection shall be enabled.
+	if *cliEnabled {
+		err := startDebug(backend)
+		if err != nil {
+			log.Fatalf("Starting debugging CLI server failed: %v", err)
+			return
+		}
+	}
+
 	// wait till node has been stopped
 	node, err := backend.NodeManager().Node()
 	if err != nil {
@@ -67,6 +79,13 @@ func main() {
 	node.Wait()
 }
 
+// startDebug starts the debugging API server.
+func startDebug(backend *api.StatusBackend) error {
+	statusAPI := api.NewStatusAPIWithBackend(backend)
+	_, err := debug.New(statusAPI, *cliPort)
+	return err
+}
+
 // makeNodeConfig parses incoming CLI options and returns node configuration object
 func makeNodeConfig() (*params.NodeConfig, error) {
 	devMode := !*prodMode
@@ -135,6 +154,7 @@ Examples:
   statusd -networkid 4  # run node on Rinkeby network
   statusd -datadir /dir # specify different dir for data
   statusd -ipc          # enable IPC for usage with "geth attach"
+  statusd -cli          # enable connection by statusd-cli on default port
 
 Options:
 `)

geth/api/api.go

@@ -16,10 +16,16 @@ type StatusAPI struct {
 	b *StatusBackend
 }
 
-// NewStatusAPI create a new StatusAPI instance
+// NewStatusAPI creates a new StatusAPI instance
 func NewStatusAPI() *StatusAPI {
+	return NewStatusAPIWithBackend(NewStatusBackend())
+}
+
+// NewStatusAPIWithBackend creates a new StatusAPI instance using
+// the passed backend.
+func NewStatusAPIWithBackend(b *StatusBackend) *StatusAPI {
 	return &StatusAPI{
-		b: NewStatusBackend(),
+		b: b,
 	}
 }