Merge pull request #2748 from hashicorp/f-operator-rework

Split operator raft command into subcommands
This commit is contained in:
Kyle Havlovitz 2017-02-16 12:07:53 -08:00 committed by GitHub
commit 1eb33e58c9
12 changed files with 530 additions and 249 deletions

View File

@ -1,13 +1,10 @@
package command
import (
"fmt"
"strings"
"flag"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/command/base"
"github.com/ryanuber/columnize"
"github.com/mitchellh/cli"
)
// OperatorCommand is used to provide various low-level tools for Consul
@ -18,7 +15,7 @@ type OperatorCommand struct {
func (c *OperatorCommand) Help() string {
helpText := `
Usage: consul operator <subcommand> [action] [options]
Usage: consul operator <subcommand> [options]
Provides cluster-level tools for Consul operators, such as interacting with
the Raft subsystem. NOTE: Use this command with extreme caution, as improper
@ -30,139 +27,15 @@ Usage: consul operator <subcommand> [action] [options]
Run consul operator <subcommand> with no arguments for help on that
subcommand.
Subcommands:
raft View and modify Consul's Raft configuration.
`
return strings.TrimSpace(helpText)
}
func (c *OperatorCommand) Run(args []string) int {
if len(args) < 1 {
c.Ui.Error("A subcommand must be specified")
c.Ui.Error("")
c.Ui.Error(c.Help())
return 1
}
var err error
subcommand := args[0]
switch subcommand {
case "raft":
err = c.raft(args[1:])
default:
err = fmt.Errorf("unknown subcommand %q", subcommand)
}
if err != nil {
c.Ui.Error(fmt.Sprintf("Operator %q subcommand failed: %v", subcommand, err))
return 1
}
return 0
return cli.RunResultHelp
}
// Synopsis returns a one-line description of this command.
func (c *OperatorCommand) Synopsis() string {
return "Provides cluster-level tools for Consul operators"
}
const raftHelp = `
Operator Raft Subcommand:
The raft subcommand can be used in two modes:
consul operator raft -list-peers
Displays the current Raft peer configuration.
consul operator raft -remove-peer -address="IP:port"
Removes Consul server with given -address from the Raft configuration.
There are rare cases where a peer may be left behind in the Raft quorum even
though the server is no longer present and known to the cluster. This
command can be used to remove the failed server so that it is no longer
affects the Raft quorum. If the server still shows in the output of the
"consul members" command, it is preferable to clean up by simply running
"consul force-leave" instead of this command.
`
// raft handles the raft subcommands.
func (c *OperatorCommand) raft(args []string) error {
f := c.Command.NewFlagSet(c)
// Parse verb arguments.
var listPeers, removePeer bool
f.BoolVar(&listPeers, "list-peers", false,
"If this flag is provided, the current Raft peer configuration will be "+
"displayed. If the cluster is in an outage state without a leader, you may need "+
"to set -stale to 'true' to get the configuration from a non-leader server.")
f.BoolVar(&removePeer, "remove-peer", false,
"If this flag is provided, the Consul server with the given -address will be "+
"removed from the Raft configuration.")
// Parse other arguments.
var address string
f.StringVar(&address, "address", "",
"The address to remove from the Raft configuration.")
if err := c.Command.Parse(args); err != nil {
if err == flag.ErrHelp {
c.Ui.Output("")
c.Ui.Output(strings.TrimSpace(raftHelp + c.Command.Help()))
return nil
}
return err
}
// Set up a client.
client, err := c.Command.HTTPClient()
if err != nil {
return fmt.Errorf("error connecting to Consul agent: %s", err)
}
operator := client.Operator()
// Dispatch based on the verb argument.
if listPeers {
// Fetch the current configuration.
q := &api.QueryOptions{
AllowStale: c.Command.HTTPStale(),
}
reply, err := operator.RaftGetConfiguration(q)
if err != nil {
return err
}
// Format it as a nice table.
result := []string{"Node|ID|Address|State|Voter"}
for _, s := range reply.Servers {
state := "follower"
if s.Leader {
state = "leader"
}
result = append(result, fmt.Sprintf("%s|%s|%s|%s|%v",
s.Node, s.ID, s.Address, state, s.Voter))
}
c.Ui.Output(columnize.SimpleFormat(result))
} else if removePeer {
// TODO (slackpad) Once we expose IDs, add support for removing
// by ID, add support for that.
if len(address) == 0 {
return fmt.Errorf("an address is required for the peer to remove")
}
// Try to kick the peer.
if err := operator.RaftRemovePeerByAddress(address, nil); err != nil {
return err
}
c.Ui.Output(fmt.Sprintf("Removed peer with address %q", address))
} else {
c.Ui.Output(c.Help())
c.Ui.Output("")
c.Ui.Output(strings.TrimSpace(raftHelp + c.Command.Help()))
}
return nil
}

100
command/operator_raft.go Normal file
View File

@ -0,0 +1,100 @@
package command
import (
"flag"
"fmt"
"strings"
"github.com/hashicorp/consul/command/base"
)
type OperatorRaftCommand struct {
base.Command
}
func (c *OperatorRaftCommand) Help() string {
helpText := `
Usage: consul operator raft <subcommand> [options]
The Raft operator command is used to interact with Consul's Raft subsystem. The
command can be used to verify Raft peers or in rare cases to recover quorum by
removing invalid peers.
Subcommands:
list-peers Display the current Raft peer configuration
remove-peer Remove a Consul server from the Raft configuration
`
return strings.TrimSpace(helpText)
}
func (c *OperatorRaftCommand) Synopsis() string {
return "Provides cluster-level tools for Consul operators"
}
func (c *OperatorRaftCommand) Run(args []string) int {
if result := c.raft(args); result != nil {
c.Ui.Error(result.Error())
return 1
}
return 0
}
// raft handles the raft subcommands.
func (c *OperatorRaftCommand) raft(args []string) error {
f := c.Command.NewFlagSet(c)
// Parse verb arguments.
var listPeers, removePeer bool
f.BoolVar(&listPeers, "list-peers", false,
"If this flag is provided, the current Raft peer configuration will be "+
"displayed. If the cluster is in an outage state without a leader, you may need "+
"to set -stale to 'true' to get the configuration from a non-leader server.")
f.BoolVar(&removePeer, "remove-peer", false,
"If this flag is provided, the Consul server with the given -address will be "+
"removed from the Raft configuration.")
// Parse other arguments.
var address string
f.StringVar(&address, "address", "",
"The address to remove from the Raft configuration.")
// Leave these flags for backwards compatibility, but hide them
// TODO: remove flags/behavior from this command in Consul 0.9
c.Command.HideFlags("list-peers", "remove-peer", "address")
if err := c.Command.Parse(args); err != nil {
if err == flag.ErrHelp {
return nil
}
return err
}
// Set up a client.
client, err := c.Command.HTTPClient()
if err != nil {
return fmt.Errorf("error connecting to Consul agent: %s", err)
}
operator := client.Operator()
// Dispatch based on the verb argument.
if listPeers {
result, err := raftListPeers(operator, c.Command.HTTPStale())
if err != nil {
c.Ui.Error(fmt.Sprintf("Error getting peers: %v", err))
}
c.Ui.Output(result)
} else if removePeer {
if err := raftRemovePeers(address, operator); err != nil {
return fmt.Errorf("Error removing peer: %v", err)
}
c.Ui.Output(fmt.Sprintf("Removed peer with address %q", address))
} else {
c.Ui.Output(c.Help())
return nil
}
return nil
}

View File

@ -0,0 +1,81 @@
package command
import (
"flag"
"fmt"
"strings"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/command/base"
"github.com/ryanuber/columnize"
)
type OperatorRaftListCommand struct {
base.Command
}
func (c *OperatorRaftListCommand) Help() string {
helpText := `
Usage: consul operator raft list-peers [options]
Displays the current Raft peer configuration.
` + c.Command.Help()
return strings.TrimSpace(helpText)
}
func (c *OperatorRaftListCommand) Synopsis() string {
return "Display the current Raft peer configuration"
}
func (c *OperatorRaftListCommand) Run(args []string) int {
c.Command.NewFlagSet(c)
if err := c.Command.Parse(args); err != nil {
if err == flag.ErrHelp {
return 0
}
c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
return 1
}
// Set up a client.
client, err := c.Command.HTTPClient()
if err != nil {
c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
return 1
}
// Fetch the current configuration.
result, err := raftListPeers(client.Operator(), c.Command.HTTPStale())
if err != nil {
c.Ui.Error(fmt.Sprintf("Error getting peers: %v", err))
}
c.Ui.Output(result)
return 0
}
func raftListPeers(operator *api.Operator, stale bool) (string, error) {
q := &api.QueryOptions{
AllowStale: stale,
}
reply, err := operator.RaftGetConfiguration(q)
if err != nil {
return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err)
}
// Format it as a nice table.
result := []string{"Node|ID|Address|State|Voter"}
for _, s := range reply.Servers {
state := "follower"
if s.Leader {
state = "leader"
}
result = append(result, fmt.Sprintf("%s|%s|%s|%s|%v",
s.Node, s.ID, s.Address, state, s.Voter))
}
return columnize.SimpleFormat(result), nil
}

View File

@ -0,0 +1,55 @@
package command
import (
"strings"
"testing"
"github.com/hashicorp/consul/command/base"
"github.com/mitchellh/cli"
)
func TestOperator_Raft_ListPeers_Implements(t *testing.T) {
var _ cli.Command = &OperatorRaftListCommand{}
}
func TestOperator_Raft_ListPeers(t *testing.T) {
a1 := testAgent(t)
defer a1.Shutdown()
waitForLeader(t, a1.httpAddr)
// Test the legacy mode with 'consul operator raft -list-peers'
{
ui, c := testOperatorRaftCommand(t)
args := []string{"-http-addr=" + a1.httpAddr, "-list-peers"}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
output := strings.TrimSpace(ui.OutputWriter.String())
if !strings.Contains(output, "leader") {
t.Fatalf("bad: %s", output)
}
}
// Test the list-peers subcommand directly
{
ui := new(cli.MockUi)
c := OperatorRaftListCommand{
Command: base.Command{
Ui: ui,
Flags: base.FlagSetHTTP,
},
}
args := []string{"-http-addr=" + a1.httpAddr}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
output := strings.TrimSpace(ui.OutputWriter.String())
if !strings.Contains(output, "leader") {
t.Fatalf("bad: %s", output)
}
}
}

View File

@ -0,0 +1,83 @@
package command
import (
"flag"
"fmt"
"strings"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/command/base"
)
type OperatorRaftRemoveCommand struct {
base.Command
}
func (c *OperatorRaftRemoveCommand) Help() string {
helpText := `
Usage: consul operator raft remove-peer [options]
Remove the Consul server with given -address from the Raft configuration.
There are rare cases where a peer may be left behind in the Raft quorum even
though the server is no longer present and known to the cluster. This command
can be used to remove the failed server so that it is no longer affects the Raft
quorum. If the server still shows in the output of the "consul members" command,
it is preferable to clean up by simply running "consul force-leave" instead of
this command.
` + c.Command.Help()
return strings.TrimSpace(helpText)
}
func (c *OperatorRaftRemoveCommand) Synopsis() string {
return "Remove a Consul server from the Raft configuration"
}
func (c *OperatorRaftRemoveCommand) Run(args []string) int {
f := c.Command.NewFlagSet(c)
var address string
f.StringVar(&address, "address", "",
"The address to remove from the Raft configuration.")
if err := c.Command.Parse(args); err != nil {
if err == flag.ErrHelp {
return 0
}
c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
return 1
}
// Set up a client.
client, err := c.Command.HTTPClient()
if err != nil {
c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
return 1
}
// Fetch the current configuration.
if err := raftRemovePeers(address, client.Operator()); err != nil {
c.Ui.Error(fmt.Sprintf("Error removing peer: %v", err))
return 1
}
c.Ui.Output(fmt.Sprintf("Removed peer with address %q", address))
return 0
}
func raftRemovePeers(address string, operator *api.Operator) error {
// TODO (slackpad) Once we expose IDs, add support for removing
// by ID, add support for that.
if len(address) == 0 {
return fmt.Errorf("an address is required for the peer to remove")
}
// Try to kick the peer.
if err := operator.RaftRemovePeerByAddress(address, nil); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,59 @@
package command
import (
"strings"
"testing"
"github.com/hashicorp/consul/command/base"
"github.com/mitchellh/cli"
)
func TestOperator_Raft_RemovePeer_Implements(t *testing.T) {
var _ cli.Command = &OperatorRaftRemoveCommand{}
}
func TestOperator_Raft_RemovePeer(t *testing.T) {
a1 := testAgent(t)
defer a1.Shutdown()
waitForLeader(t, a1.httpAddr)
// Test the legacy mode with 'consul operator raft -remove-peer'
{
ui, c := testOperatorRaftCommand(t)
args := []string{"-http-addr=" + a1.httpAddr, "-remove-peer", "-address=nope"}
code := c.Run(args)
if code != 1 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
// If we get this error, it proves we sent the address all they through.
output := strings.TrimSpace(ui.ErrorWriter.String())
if !strings.Contains(output, "address \"nope\" was not found in the Raft configuration") {
t.Fatalf("bad: %s", output)
}
}
// Test the remove-peer subcommand directly
{
ui := new(cli.MockUi)
c := OperatorRaftRemoveCommand{
Command: base.Command{
Ui: ui,
Flags: base.FlagSetHTTP,
},
}
args := []string{"-http-addr=" + a1.httpAddr, "-address=nope"}
code := c.Run(args)
if code != 1 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
// If we get this error, it proves we sent the address all they through.
output := strings.TrimSpace(ui.ErrorWriter.String())
if !strings.Contains(output, "address \"nope\" was not found in the Raft configuration") {
t.Fatalf("bad: %s", output)
}
}
}

View File

@ -0,0 +1,22 @@
package command
import (
"testing"
"github.com/hashicorp/consul/command/base"
"github.com/mitchellh/cli"
)
func testOperatorRaftCommand(t *testing.T) (*cli.MockUi, *OperatorRaftCommand) {
ui := new(cli.MockUi)
return ui, &OperatorRaftCommand{
Command: base.Command{
Ui: ui,
Flags: base.FlagSetHTTP,
},
}
}
func TestOperator_Raft_Implements(t *testing.T) {
var _ cli.Command = &OperatorRaftCommand{}
}

View File

@ -1,61 +1,11 @@
package command
import (
"strings"
"testing"
"github.com/hashicorp/consul/command/base"
"github.com/mitchellh/cli"
)
func testOperatorCommand(t *testing.T) (*cli.MockUi, *OperatorCommand) {
ui := new(cli.MockUi)
return ui, &OperatorCommand{
Command: base.Command{
Ui: ui,
Flags: base.FlagSetHTTP,
},
}
}
func TestOperator_Implements(t *testing.T) {
var _ cli.Command = &OperatorCommand{}
}
func TestOperator_Raft_ListPeers(t *testing.T) {
a1 := testAgent(t)
defer a1.Shutdown()
waitForLeader(t, a1.httpAddr)
ui, c := testOperatorCommand(t)
args := []string{"raft", "-http-addr=" + a1.httpAddr, "-list-peers"}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
output := strings.TrimSpace(ui.OutputWriter.String())
if !strings.Contains(output, "leader") {
t.Fatalf("bad: %s", output)
}
}
func TestOperator_Raft_RemovePeer(t *testing.T) {
a1 := testAgent(t)
defer a1.Shutdown()
waitForLeader(t, a1.httpAddr)
ui, c := testOperatorCommand(t)
args := []string{"raft", "-http-addr=" + a1.httpAddr, "-remove-peer", "-address=nope"}
code := c.Run(args)
if code != 1 {
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
}
// If we get this error, it proves we sent the address all they through.
output := strings.TrimSpace(ui.ErrorWriter.String())
if !strings.Contains(output, "address \"nope\" was not found in the Raft configuration") {
t.Fatalf("bad: %s", output)
}
}

View File

@ -206,6 +206,33 @@ func init() {
"operator": func() (cli.Command, error) {
return &command.OperatorCommand{
Command: base.Command{
Flags: base.FlagSetNone,
Ui: ui,
},
}, nil
},
"operator raft": func() (cli.Command, error) {
return &command.OperatorRaftCommand{
Command: base.Command{
Flags: base.FlagSetHTTP,
Ui: ui,
},
}, nil
},
"operator raft list-peers": func() (cli.Command, error) {
return &command.OperatorRaftListCommand{
Command: base.Command{
Flags: base.FlagSetHTTP,
Ui: ui,
},
}, nil
},
"operator raft remove-peer": func() (cli.Command, error) {
return &command.OperatorRaftRemoveCommand{
Command: base.Command{
Flags: base.FlagSetHTTP,
Ui: ui,

View File

@ -28,72 +28,17 @@ endpoint.
## Usage
Usage: `consul operator <subcommand> [action] [options]`
```text
Usage: consul operator <subcommand> [options]
Run `consul operator <subcommand>` with no arguments for help on that
subcommand. The following subcommands are available:
# ...
* `raft` - View and modify Consul's Raft configuration.
Subcommands:
#### API Options
<%= partial "docs/commands/http_api_options_client" %>
<%= partial "docs/commands/http_api_options_server" %>
## Raft Operations
The `raft` subcommand is used to view and modify Consul's Raft configuration.
Two actions are available, as detailed in this section.
<a name="raft-list-peers"></a>
#### Display Peer Configuration
This action displays the current Raft peer configuration.
Usage: `consul operator raft -list-peers -stale=[true|false]`
* `-stale` - Optional and defaults to "false" which means the leader provides
the result. If the cluster is in an outage state without a leader, you may need
to set this to "true" to get the configuration from a non-leader server.
The output looks like this:
```
Node ID Address State Voter
alice 127.0.0.1:8300 127.0.0.1:8300 follower true
bob 127.0.0.2:8300 127.0.0.2:8300 leader true
carol 127.0.0.3:8300 127.0.0.3:8300 follower true
raft Provides cluster-level tools for Consul operators
```
`Node` is the node name of the server, as known to Consul, or "(unknown)" if
the node is stale and not known.
For more information, examples, and usage about a subcommand, click on the name
of the subcommand in the sidebar or one of the links below:
`ID` is the ID of the server. This is the same as the `Address` in Consul 0.7
but may be upgraded to a GUID in a future version of Consul.
`Address` is the IP:port for the server.
`State` is either "follower" or "leader" depending on the server's role in the
Raft configuration.
`Voter` is "true" or "false", indicating if the server has a vote in the Raft
configuration. Future versions of Consul may add support for non-voting servers.
<a name="raft-remove-peer"></a>
#### Remove a Peer
This command removes Consul server with given address from the Raft configuration.
There are rare cases where a peer may be left behind in the Raft configuration
even though the server is no longer present and known to the cluster. This command
can be used to remove the failed server so that it is no longer affects the
Raft quorum. If the server still shows in the output of the
[`consul members`](/docs/commands/members.html) command, it is preferable to
clean up by simply running
[`consul force-leave`](/docs/commands/force-leave.html)
instead of this command.
Usage: `consul operator raft -remove-peer -address="IP:port"`
* `-address` - "IP:port" for the server to remove. The port number is usually
8300, unless configured otherwise.
The return code will indicate success or failure.
- [raft] (/docs/commands/operator/raft.html)

View File

@ -0,0 +1,81 @@
---
layout: "docs"
page_title: "Commands: Operator Raft"
sidebar_current: "docs-commands-operator-raft"
description: >
The operator raft subcommand is used to view and modify Consul's Raft configuration.
---
# Consul Operator Raft
Command: `consul operator raft`
The Raft operator command is used to interact with Consul's Raft subsystem. The
command can be used to verify Raft peers or in rare cases to recover quorum by
removing invalid peers.
```text
Usage: consul operator raft <subcommand> [options]
The Raft operator command is used to interact with Consul's Raft subsystem. The
command can be used to verify Raft peers or in rare cases to recover quorum by
removing invalid peers.
Subcommands:
list-peers Display the current Raft peer configuration
remove-peer Remove a Consul server from the Raft configuration
```
## list-peers
This command displays the current Raft peer configuration.
Usage: `consul operator raft list-peers -stale=[true|false]`
* `-stale` - Optional and defaults to "false" which means the leader provides
the result. If the cluster is in an outage state without a leader, you may need
to set this to "true" to get the configuration from a non-leader server.
The output looks like this:
```
Node ID Address State Voter
alice 127.0.0.1:8300 127.0.0.1:8300 follower true
bob 127.0.0.2:8300 127.0.0.2:8300 leader true
carol 127.0.0.3:8300 127.0.0.3:8300 follower true
```
`Node` is the node name of the server, as known to Consul, or "(unknown)" if
the node is stale and not known.
`ID` is the ID of the server. This is the same as the `Address` in Consul 0.7
but may be upgraded to a GUID in a future version of Consul.
`Address` is the IP:port for the server.
`State` is either "follower" or "leader" depending on the server's role in the
Raft configuration.
`Voter` is "true" or "false", indicating if the server has a vote in the Raft
configuration. Future versions of Consul may add support for non-voting servers.
## remove-peer
This command removes the Consul server with given address from the Raft configuration.
There are rare cases where a peer may be left behind in the Raft configuration
even though the server is no longer present and known to the cluster. This command
can be used to remove the failed server so that it is no longer affects the
Raft quorum. If the server still shows in the output of the
[`consul members`](/docs/commands/members.html) command, it is preferable to
clean up by simply running
[`consul force-leave`](/docs/commands/force-leave.html)
instead of this command.
Usage: `consul operator raft remove-peer -address="IP:port"`
* `-address` - "IP:port" for the server to remove. The port number is usually
8300, unless configured otherwise.
The return code will indicate success or failure.

View File

@ -15,7 +15,7 @@
<li<%= sidebar_current("docs-upgrading-compat") %>>
<a href="/docs/compatibility.html">Compatibility Promise</a>
</li>
</li>
<li<%= sidebar_current("docs-upgrading-specific") %>>
<a href="/docs/upgrade-specific.html">Specific Version Details</a>
@ -141,6 +141,11 @@
<li<%= sidebar_current("docs-commands-operator") %>>
<a href="/docs/commands/operator.html">operator</a>
<ul class="subnav">
<li<%= sidebar_current("docs-commands-operator-raft") %>>
<a href="/docs/commands/operator/raft.html">raft</a>
</li>
</ul>
</li>
<li<%= sidebar_current("docs-commands-reload") %>>
@ -279,7 +284,7 @@
<li<%= sidebar_current("docs-guides") %>>
<a href="/docs/guides/index.html">Guides</a>
<ul class="nav">
<ul class="nav">
<li<%= sidebar_current("docs-guides-atlas") %>>
<a href="/docs/guides/atlas.html">Atlas Integration</a>
</li>
@ -314,16 +319,16 @@
<li<%= sidebar_current("docs-guides-outage") %>>
<a href="/docs/guides/outage.html">Outage Recovery</a>
</li>
</li>
<li<%= sidebar_current("docs-guides-semaphore") %>>
<li<%= sidebar_current("docs-guides-semaphore") %>>
<a href="/docs/guides/semaphore.html">Semaphore</a>
</li>
</li>
<li<%= sidebar_current("docs-guides-performance") %>>
<a href="/docs/guides/performance.html">Server Performance</a>
</li>
</ul>
</ul>
<li<%= sidebar_current("docs-faq") %>>
<a href="/docs/faq.html">Frequently Asked Questions</a>