From 227019ec598db5f2a8277f0a3f04ec132a335112 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:07:00 -0500 Subject: [PATCH 01/22] Add documentation to KVPair Based on discussions with @slackpad, some of the values in the KVPair are ReadOnly. This commit updates the docs to reflect that. --- api/kv.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/api/kv.go b/api/kv.go index 3dac2583c1..bbd49813c7 100644 --- a/api/kv.go +++ b/api/kv.go @@ -11,13 +11,33 @@ import ( // KVPair is used to represent a single K/V entry type KVPair struct { - Key string + // Key is the name of the key. It is also part of the URL path when accessed + // via the API. + Key string + + // CreateIndex holds the index corresponding the creation of this KVPair. This + // is a read-only field. CreateIndex uint64 + + // ModifyIndex is used for the Check-And-Set operations. ModifyIndex uint64 - LockIndex uint64 - Flags uint64 - Value []byte - Session string + + // LockIndex holds the index corresponding to a lock on this key, if any. This + // is a read-only field. + LockIndex uint64 + + // Flags are any user-defined flags on the key. It is up to the implementer + // to check these values, since Consul does not treat them specially. + Flags uint64 + + // Value is the value for the key. This can be any value, but it will be + // base64 encoded upon transport. + Value []byte + + // Session is a string representing the name of the session. Any other + // interactions with this key over the same session must specify the same + // session name. + Session string } // KVPairs is a list of KVPair objects From 8c30edc54ef54eb026c6c88db3c38edf7cbfa5a2 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:08:47 -0500 Subject: [PATCH 02/22] Add test function for returning an API client --- command/util_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/command/util_test.go b/command/util_test.go index a48f33cb0c..e06492aa02 100644 --- a/command/util_test.go +++ b/command/util_test.go @@ -2,8 +2,6 @@ package command import ( "fmt" - "github.com/hashicorp/consul/command/agent" - "github.com/hashicorp/consul/consul" "io" "io/ioutil" "math/rand" @@ -12,6 +10,10 @@ import ( "sync/atomic" "testing" "time" + + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/command/agent" + "github.com/hashicorp/consul/consul" ) var offset uint64 @@ -42,6 +44,15 @@ func testAgent(t *testing.T) *agentWrapper { return testAgentWithConfig(t, func(c *agent.Config) {}) } +func testAgentWithAPIClient(t *testing.T) (*agentWrapper, *api.Client) { + agent := testAgentWithConfig(t, func(c *agent.Config) {}) + client, err := api.NewClient(&api.Config{Address: agent.httpAddr}) + if err != nil { + t.Fatalf("consul client: %#v", err) + } + return agent, client +} + func testAgentWithConfig(t *testing.T, cb func(c *agent.Config)) *agentWrapper { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { From afd0e58c2405b812605e3bba56f52297a6e6eded Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:09:33 -0500 Subject: [PATCH 03/22] Update middleman --- website/Gemfile | 2 +- website/Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/website/Gemfile b/website/Gemfile index 7ef6a8f31f..7d30aae647 100644 --- a/website/Gemfile +++ b/website/Gemfile @@ -1,3 +1,3 @@ source "https://rubygems.org" -gem "middleman-hashicorp", github: "hashicorp/middleman-hashicorp" +gem "middleman-hashicorp", git: "https://github.com/hashicorp/middleman-hashicorp.git" diff --git a/website/Gemfile.lock b/website/Gemfile.lock index 1b5a4a6060..8925928e23 100644 --- a/website/Gemfile.lock +++ b/website/Gemfile.lock @@ -1,5 +1,5 @@ GIT - remote: git://github.com/hashicorp/middleman-hashicorp.git + remote: https://github.com/hashicorp/middleman-hashicorp.git revision: 80ddc227b26cbbb3742d14396f26172174222080 specs: middleman-hashicorp (0.2.0) @@ -191,4 +191,4 @@ DEPENDENCIES middleman-hashicorp! BUNDLED WITH - 1.11.2 + 1.13.1 From 87ad6d466c3bd82669626518f49f49a84cff84f3 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:10:58 -0500 Subject: [PATCH 04/22] Add kv command stubs --- command/kv_command.go | 76 ++++++++++++++++++++++++++++++++++++++ command/kv_command_test.go | 15 ++++++++ command/util_test.go | 8 ++++ commands.go | 6 +++ 4 files changed, 105 insertions(+) create mode 100644 command/kv_command.go create mode 100644 command/kv_command_test.go diff --git a/command/kv_command.go b/command/kv_command.go new file mode 100644 index 0000000000..28bd71a942 --- /dev/null +++ b/command/kv_command.go @@ -0,0 +1,76 @@ +package command + +import ( + "strings" + + "github.com/mitchellh/cli" +) + +// KVCommand is a Command implementation that just shows help for +// the subcommands nested below it. +type KVCommand struct { + Ui cli.Ui +} + +func (c *KVCommand) Run(args []string) int { + return cli.RunResultHelp +} + +func (c *KVCommand) Help() string { + helpText := ` +Usage: consul kv [options] [args] + + This command has subcommands for interacting with Consul's key-value + store. Here are some simple examples, and more detailed examples are + available in the subcommands or the documentation. + + Create or update the key named "redis/config/connections" with the value "5": + + $ consul kv put redis/config/connections 5 + + Read this value back: + + $ consul kv get redis/config/connections + + Or get detailed key information: + + $ consul kv get -detailed redis/config/connections + + Finally, delete the key: + + $ consul kv delete redis/config/connections + + For more examples, ask for subcommand help or view the documentation. + +` + return strings.TrimSpace(helpText) +} + +func (c *KVCommand) Synopsis() string { + return "Interact with the key-value store" +} + +var apiOptsText = strings.TrimSpace(` +API Options: + + -http-addr= Address of the Consul agent with the port. This can + be an IP address or DNS address, but it must include + the port. This can also be specified via the + CONSUL_HTTP_ADDR environment variable. The default + value is 127.0.0.1:8500. + + -datacenter= Name of the datacenter to query. If unspecified, the + query will default to the datacenter of the Consul + agent at the HTTP address. + + -token= ACL token to use in the request. This can also be + specified via the CONSUL_HTTP_TOKEN environment + variable. If unspecified, the query will default to + the token of the Consul agent at the HTTP address. + + -stale Permit any Consul server (non-leader) to respond to + this request. This allows for lower latency and higher + throughput, but can result in stale data. This option + has no effect on non-read operations. The default + value is false. +`) diff --git a/command/kv_command_test.go b/command/kv_command_test.go new file mode 100644 index 0000000000..9509381082 --- /dev/null +++ b/command/kv_command_test.go @@ -0,0 +1,15 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" +) + +func TestKVCommand_implements(t *testing.T) { + var _ cli.Command = &KVCommand{} +} + +func TestKVCommand_noTabs(t *testing.T) { + assertNoTabs(t, new(KVCommand)) +} diff --git a/command/util_test.go b/command/util_test.go index e06492aa02..faf0377bc6 100644 --- a/command/util_test.go +++ b/command/util_test.go @@ -7,6 +7,7 @@ import ( "math/rand" "net" "os" + "strings" "sync/atomic" "testing" "time" @@ -14,6 +15,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/command/agent" "github.com/hashicorp/consul/consul" + "github.com/mitchellh/cli" ) var offset uint64 @@ -137,3 +139,9 @@ func nextConfig() *agent.Config { return conf } + +func assertNoTabs(t *testing.T, c cli.Command) { + if strings.ContainsRune(c.Help(), '\t') { + t.Errorf("%#v help output contains tabs", c) + } +} diff --git a/commands.go b/commands.go index 2a25c77f81..691ac8de6c 100644 --- a/commands.go +++ b/commands.go @@ -53,6 +53,12 @@ func init() { }, nil }, + "kv": func() (cli.Command, error) { + return &command.KVCommand{ + Ui: ui, + }, nil + }, + "join": func() (cli.Command, error) { return &command.JoinCommand{ Ui: ui, From c8fdc5c50d2295ebe3806181ef851829cbbf6a5c Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:12:14 -0500 Subject: [PATCH 05/22] Add kv get --- command/kv_get.go | 226 ++++++++++++++++ command/kv_get_test.go | 252 ++++++++++++++++++ commands.go | 6 + .../commands/_http_api_options.html.markdown | 16 ++ .../source/docs/commands/index.html.markdown | 3 + .../docs/commands/kv/get.html.markdown.erb | 150 +++++++++++ website/source/layouts/docs.erb | 9 + 7 files changed, 662 insertions(+) create mode 100644 command/kv_get.go create mode 100644 command/kv_get_test.go create mode 100644 website/source/docs/commands/_http_api_options.html.markdown create mode 100644 website/source/docs/commands/kv/get.html.markdown.erb diff --git a/command/kv_get.go b/command/kv_get.go new file mode 100644 index 0000000000..0358d7085f --- /dev/null +++ b/command/kv_get.go @@ -0,0 +1,226 @@ +package command + +import ( + "bytes" + "flag" + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/hashicorp/consul/api" + "github.com/mitchellh/cli" +) + +// KVGetCommand is a Command implementation that is used to setup +// a "watch" which uses a sub-process +type KVGetCommand struct { + Ui cli.Ui +} + +func (c *KVGetCommand) Help() string { + helpText := ` +Usage: consul kv get [options] [KEY_OR_PREFIX] + + Retrieves the value from Consul's key-value store at the given key name. If no + key exists with that name, an error is returned. If a key exists with that + name but has no data, nothing is returned. If the name or prefix is omitted, + it defaults to "" which is the root of the key-value store. + + To retrieve the value for the key named "foo" in the key-value store: + + $ consul kv get foo + + This will return the original, raw value stored in Consul. To view detailed + information about the key, specify the "-detailed" flag. This will output all + known metadata about the key including ModifyIndex and any user-supplied + flags: + + $ consul kv get -detailed foo + + To treat the path as a prefix and list all keys which start with the given + prefix, specify the "-recurse" flag: + + $ consul kv get -recurse foo + + This will return all key-vlaue pairs. To just list the keys which start with + the specified prefix, use the "-keys" option instead: + + $ consul kv get -keys foo + + For a full list of options and examples, please see the Consul documentation. + +` + apiOptsText + ` + +KV Get Options: + + -detailed Provide additional metadata about the key in addition + to the value such as the ModifyIndex and any flags + that may have been set on the key. The default value + is false. + + -keys List keys which start with the given prefix, but not + their values. This is especially useful if you only + need the key names themselves. This option is commonly + combined with the -separator option. The default value + is false. + + -recurse Recursively look at all keys prefixed with the given + path. The default value is false. + + -separator= String to use as a separator between keys. The default + value is "/", but this option is only taken into + account when paired with the -keys flag. + +` + return strings.TrimSpace(helpText) +} + +func (c *KVGetCommand) Run(args []string) int { + cmdFlags := flag.NewFlagSet("get", flag.ContinueOnError) + cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } + datacenter := cmdFlags.String("datacenter", "", "") + token := cmdFlags.String("token", "", "") + stale := cmdFlags.Bool("stale", false, "") + detailed := cmdFlags.Bool("detailed", false, "") + keys := cmdFlags.Bool("keys", false, "") + recurse := cmdFlags.Bool("recurse", false, "") + separator := cmdFlags.String("separator", "/", "") + httpAddr := HTTPAddrFlag(cmdFlags) + if err := cmdFlags.Parse(args); err != nil { + return 1 + } + + key := "" + + // Check for arg validation + args = cmdFlags.Args() + switch len(args) { + case 0: + key = "" + case 1: + key = args[0] + default: + c.Ui.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args))) + return 1 + } + + // This is just a "nice" thing to do. Since pairs cannot start with a /, but + // users will likely put "/" or "/foo", lets go ahead and strip that for them + // here. + if len(key) > 0 && key[0] == '/' { + key = key[1:] + } + + // If the key is empty and we are not doing a recursive or key-based lookup, + // this is an error. + if key == "" && !(*recurse || *keys) { + c.Ui.Error("Error! Missing KEY argument") + return 1 + } + + // Create and test the HTTP client + conf := api.DefaultConfig() + conf.Address = *httpAddr + conf.Token = *token + client, err := api.NewClient(conf) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) + return 1 + } + + switch { + case *keys: + keys, _, err := client.KV().Keys(key, *separator, &api.QueryOptions{ + Datacenter: *datacenter, + AllowStale: *stale, + }) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying Consul agent: %s", err)) + return 1 + } + + for _, k := range keys { + c.Ui.Info(string(k)) + } + + return 0 + case *recurse: + pairs, _, err := client.KV().List(key, &api.QueryOptions{ + Datacenter: *datacenter, + AllowStale: *stale, + }) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying Consul agent: %s", err)) + return 1 + } + + for i, pair := range pairs { + if *detailed { + var b bytes.Buffer + if err := prettyKVPair(&b, pair); err != nil { + c.Ui.Error(fmt.Sprintf("Error rendering KV pair: %s", err)) + return 1 + } + + c.Ui.Info(b.String()) + + if i < len(pairs)-1 { + c.Ui.Info("") + } + } else { + c.Ui.Info(fmt.Sprintf("%s:%s", pair.Key, pair.Value)) + } + } + + return 0 + default: + pair, _, err := client.KV().Get(key, &api.QueryOptions{ + Datacenter: *datacenter, + AllowStale: *stale, + }) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying Consul agent: %s", err)) + return 1 + } + + if pair == nil { + c.Ui.Error(fmt.Sprintf("Error! No key exists at: %s", key)) + return 1 + } + + if *detailed { + var b bytes.Buffer + if err := prettyKVPair(&b, pair); err != nil { + c.Ui.Error(fmt.Sprintf("Error rendering KV pair: %s", err)) + return 1 + } + + c.Ui.Info(b.String()) + return 0 + } else { + c.Ui.Info(string(pair.Value)) + return 0 + } + } +} + +func (c *KVGetCommand) Synopsis() string { + return "Retrieves or lists data from the KV store" +} + +func prettyKVPair(w io.Writer, pair *api.KVPair) error { + tw := tabwriter.NewWriter(w, 0, 2, 6, ' ', 0) + fmt.Fprintf(tw, "CreateIndex\t%d\n", pair.CreateIndex) + fmt.Fprintf(tw, "Flags\t%d\n", pair.Flags) + fmt.Fprintf(tw, "Key\t%s\n", pair.Key) + fmt.Fprintf(tw, "LockIndex\t%d\n", pair.LockIndex) + fmt.Fprintf(tw, "ModifyIndex\t%d\n", pair.ModifyIndex) + if pair.Session == "" { + fmt.Fprintf(tw, "Session\t-\n") + } else { + fmt.Fprintf(tw, "Session\t%s\n", pair.Session) + } + fmt.Fprintf(tw, "Value\t%s", pair.Value) + return tw.Flush() +} diff --git a/command/kv_get_test.go b/command/kv_get_test.go new file mode 100644 index 0000000000..e0dc8ce31a --- /dev/null +++ b/command/kv_get_test.go @@ -0,0 +1,252 @@ +package command + +import ( + "strings" + "testing" + + "github.com/hashicorp/consul/api" + "github.com/mitchellh/cli" +) + +func TestKVGetCommand_implements(t *testing.T) { + var _ cli.Command = &KVGetCommand{} +} + +func TestKVGetCommand_noTabs(t *testing.T) { + assertNoTabs(t, new(KVGetCommand)) +} + +func TestKVGetCommand_Validation(t *testing.T) { + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + cases := map[string]struct { + args []string + output string + }{ + "no key": { + []string{}, + "Missing KEY argument", + }, + "extra args": { + []string{"foo", "bar", "baz"}, + "Too many arguments", + }, + } + + for name, tc := range cases { + // Ensure our buffer is always clear + if ui.ErrorWriter != nil { + ui.ErrorWriter.Reset() + } + if ui.OutputWriter != nil { + ui.OutputWriter.Reset() + } + + code := c.Run(tc.args) + if code == 0 { + t.Errorf("%s: expected non-zero exit", name) + } + + output := ui.ErrorWriter.String() + if !strings.Contains(output, tc.output) { + t.Errorf("%s: expected %q to contain %q", name, output, tc.output) + } + } +} + +func TestKVGetCommand_Run(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + pair := &api.KVPair{ + Key: "foo", + Value: []byte("bar"), + } + _, err := client.KV().Put(pair, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "foo", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + output := ui.OutputWriter.String() + if !strings.Contains(output, "bar") { + t.Errorf("bad: %#v", output) + } +} + +func TestKVGetCommand_Missing(t *testing.T) { + srv, _ := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + args := []string{ + "-http-addr=" + srv.httpAddr, + "not-a-real-key", + } + + code := c.Run(args) + if code == 0 { + t.Fatalf("expected bad code") + } +} + +func TestKVGetCommand_Empty(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + pair := &api.KVPair{ + Key: "empty", + Value: []byte(""), + } + _, err := client.KV().Put(pair, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "empty", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } +} + +func TestKVGetCommand_Detailed(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + pair := &api.KVPair{ + Key: "foo", + Value: []byte("bar"), + } + _, err := client.KV().Put(pair, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-detailed", + "foo", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + output := ui.OutputWriter.String() + for _, key := range []string{ + "CreateIndex", + "LockIndex", + "ModifyIndex", + "Flags", + "Session", + "Value", + } { + if !strings.Contains(output, key) { + t.Fatalf("bad %#v, missing %q", output, key) + } + } +} + +func TestKVGetCommand_Keys(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + keys := []string{"foo/bar", "foo/baz", "foo/zip"} + for _, key := range keys { + if _, err := client.KV().Put(&api.KVPair{Key: key}, nil); err != nil { + t.Fatalf("err: %#v", err) + } + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-keys", + "foo/", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + output := ui.OutputWriter.String() + for _, key := range keys { + if !strings.Contains(output, key) { + t.Fatalf("bad %#v missing %q", output, key) + } + } +} + +func TestKVGetCommand_Recurse(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVGetCommand{Ui: ui} + + keys := map[string]string{ + "foo/a": "a", + "foo/b": "b", + "foo/c": "c", + } + for k, v := range keys { + pair := &api.KVPair{Key: k, Value: []byte(v)} + if _, err := client.KV().Put(pair, nil); err != nil { + t.Fatalf("err: %#v", err) + } + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-recurse", + "foo", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + output := ui.OutputWriter.String() + for key, value := range keys { + if !strings.Contains(output, key+":"+value) { + t.Fatalf("bad %#v missing %q", output, key+"adafa") + } + } +} diff --git a/commands.go b/commands.go index 691ac8de6c..7b8c37775f 100644 --- a/commands.go +++ b/commands.go @@ -59,6 +59,12 @@ func init() { }, nil }, + "kv get": func() (cli.Command, error) { + return &command.KVGetCommand{ + Ui: ui, + }, nil + }, + "join": func() (cli.Command, error) { return &command.JoinCommand{ Ui: ui, diff --git a/website/source/docs/commands/_http_api_options.html.markdown b/website/source/docs/commands/_http_api_options.html.markdown new file mode 100644 index 0000000000..ad5e02a0c3 --- /dev/null +++ b/website/source/docs/commands/_http_api_options.html.markdown @@ -0,0 +1,16 @@ +* `-http-addr=` - Address of the Consul agent with the port. This can be + an IP address or DNS address, but it must include the port. This can also be + specified via the CONSUL_HTTP_ADDR environment variable. The default value is + 127.0.0.1:8500. + +* `-datacenter=` - Name of the datacenter to query. If unspecified, the + query will default to the datacenter of the Consul agent at the HTTP address. + +* `-token=` - ACL token to use in the request. This can also be specified + via the `CONSUL_HTTP_TOKEN` environment variable. If unspecified, the query + will default to the token of the Consul agent at the HTTP address. + +* `-stale` - Permit any Consul server (non-leader) to respond to this request. + This allows for lower latency and higher throughput, but can result in stale + data. This option has no effect on non-read operations. The default value is + false. diff --git a/website/source/docs/commands/index.html.markdown b/website/source/docs/commands/index.html.markdown index 7e0302754d..7ee03658fb 100644 --- a/website/source/docs/commands/index.html.markdown +++ b/website/source/docs/commands/index.html.markdown @@ -27,6 +27,7 @@ usage: consul [--version] [--help] [] Available commands are: agent Runs a Consul agent + configtest Validate config file event Fire a new event exec Executes a command on Consul nodes force-leave Forces a member of the cluster to enter the "left" state @@ -34,8 +35,10 @@ Available commands are: join Tell Consul agent to join cluster keygen Generates a new encryption key keyring Manages gossip layer encryption keys + kv Interact with the key-value store leave Gracefully leaves the Consul cluster and shuts down lock Execute a command holding a lock + maint Controls node or service maintenance mode members Lists the members of a Consul cluster monitor Stream logs from a Consul agent operator Provides cluster-level tools for Consul operators diff --git a/website/source/docs/commands/kv/get.html.markdown.erb b/website/source/docs/commands/kv/get.html.markdown.erb new file mode 100644 index 0000000000..29b36959da --- /dev/null +++ b/website/source/docs/commands/kv/get.html.markdown.erb @@ -0,0 +1,150 @@ +--- +layout: "docs" +page_title: "Commands: KV Get" +sidebar_current: "docs-commands-kv-get" +--- + +# Consul KV Get + +Command: `consul kv get` + +The `kv get` command is used to retrieves the value from Consul's key-value +store at the given key name. If no key exists with that name, an error is +returned. If a key exists with that name but has no data, nothing is returned. +If the name or prefix is omitted, it defaults to "" which is the root of the +key-value store. + +## Usage + +Usage: `consul kv get [options] [KEY_OR_PREFIX]` + +#### API Options + +<%= partial "docs/commands/http_api_options" %> + +#### KV Get Options + +* `-detailed` - Provide additional metadata about the key in addition to the + value such as the ModifyIndex and any flags that may have been set on the key. + The default value is false. + +* `-keys` - List keys which start with the given prefix, but not their values. + This is especially useful if you only need the key names themselves. This + option is commonly combined with the -separator option. The default value is + false. + +* `-recurse` - Recursively look at all keys prefixed with the given path. The + default value is false. + +* `-separator=` - String to use as a separator between keys. The default + value is "/", but this option is only taken into account when paired with the + -keys flag. + +## Examples + +To retrieve the value for the key named "redis/config/connections" in the +key-value store: + +``` +$ consul kv get redis/config/connections +5 +``` + +This will return the original, raw value stored in Consul. To view detailed +information about the key, specify the "-detailed" flag. This will output all +known metadata about the key including ModifyIndex and any user-supplied +flags: + +``` +$ consul kv get -detailed redis/config/connections +CreateIndex 336 +Flags 0 +Key redis/config/connections +LockIndex 0 +ModifyIndex 336 +Session - +Value 5 +``` + +If the key with the given name does not exist, an error is returned: + +``` +$ consul kv get not-a-real-key +Error! No key exists at: not-a-real-key +``` + +To treat the path as a prefix and list all keys which start with the given +prefix, specify the "-recurse" flag: + +``` +$ consul kv get -recurse redis/ +redis/config/connections:5 +redis/config/cpu:128 +redis/config/memory:512 +``` + +Or list detailed information about all pairs under a prefix: + +``` +$ consul kv get -recurse -detailed redis +CreateIndex 336 +Flags 0 +Key redis/config/connections +LockIndex 0 +ModifyIndex 336 +Session - +Value 5 + +CreateIndex 472 +Flags 0 +Key redis/config/cpu +LockIndex 0 +ModifyIndex 472 +Session - +Value 128 + +CreateIndex 471 +Flags 0 +Key redis/config/memory +LockIndex 0 +ModifyIndex 471 +Session - +Value 512 +``` + +To just list the keys which start with the specified prefix, use the "-keys" +option instead. This is more performant and results in a smaller payload: + +``` +$ consul kv get -keys redis/config/ +redis/config/connections +redis/config/cpu +redis/config/memory +``` + +By default, the `-keys` operation uses a separator of "/", meaning it will not +recurse beyond that separator. You can choose a different separator by setting +`-separator=""`. + +``` +$ consul kv get -keys -separator="s" redis +redis/c +``` + +Alternatively, you can disable the separator altogether by setting it to the +empty string: + +``` +$ consul kv get -keys -separator="" redis +redis/config/connections +redis/config/cpu +redis/config/memory +``` + +To list all keys at the root, simply omit the prefix parameter: + +``` +$ consul kv get -keys +memcached/ +redis/ +``` diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 5c02bc7dea..2738d14b43 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -99,6 +99,15 @@ keyring + > + kv +
    + > + get + +
+ + > leave From 82bddd7f9b7bbecf15cf33bffc23b9c8c2578253 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:12:29 -0500 Subject: [PATCH 06/22] Add kv put command --- command/kv_put.go | 249 ++++++++++++++ command/kv_put_test.go | 309 ++++++++++++++++++ commands.go | 6 + .../docs/commands/kv/put.html.markdown.erb | 144 ++++++++ website/source/layouts/docs.erb | 3 + 5 files changed, 711 insertions(+) create mode 100644 command/kv_put.go create mode 100644 command/kv_put_test.go create mode 100644 website/source/docs/commands/kv/put.html.markdown.erb diff --git a/command/kv_put.go b/command/kv_put.go new file mode 100644 index 0000000000..363571c3d6 --- /dev/null +++ b/command/kv_put.go @@ -0,0 +1,249 @@ +package command + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/hashicorp/consul/api" + "github.com/mitchellh/cli" +) + +// KVPutCommand is a Command implementation that is used to setup +// a "watch" which uses a sub-process +type KVPutCommand struct { + Ui cli.Ui + + // testStdin is the input for testing. + testStdin io.Reader +} + +func (c *KVPutCommand) Help() string { + helpText := ` +Usage: consul kv put [options] KEY [DATA] + + Writes the data to the given path in the key-value store. The data can be of + any type, but it will be transported as a base64-encoded string for safe + transport: + + $ consul kv put config/redis/maxconns 5 + + The data can also be consumed from a file on disk by prefixing with the "@" + symbol. For example: + + $ consul kv put config/program/license @license.lic + + The DATA argument itself is optional. If omitted, this will create an empty + key-value pair at the specified path: + + $ consul kv put webapp/beta/active + + To perform a Check-And-Set operation, specify the -cas flag with the + appropriate -modify-index flag corresponding to the key you want to perform + the CAS operation on: + + $ consul kv put -cas -modify-index=844 config/redis/maxconns 5 + + Additional flags and more advanced use cases are detailed below. + +` + apiOptsText + ` + +KV Put Options: + + -acquire Obtain a lock on the key. If the key does not exist, + this operation will create the key and obtain the + lock. The session must already exist and be specified + via the -session flag. The default value is false. + + -cas Perform a Check-And-Set operation. If this value is + specified without -modify-index, the key will first be + fetched and the resulting ModifyIndex will be used on + the next query. The default value is false. + + -flags= Unsigned integer value to assign to this key-value + pair. This value is not read by Consul, so clients can + use this value however makes sense for their use case. + The default value is 0 (no flags). + + -modify-index= Unsigned integer representing the ModifyIndex of the + key. This is often combined with the -cas flag, but it + can be specified for any key. The default value is 0. + + -release Forfeit the lock on the key at the givne path. This + requires the -session flag to be set. The key must be + held by the session in order to be unlocked. The + default value is false. + + -session= User-defined identifer for this session as a string. + This is commonly used with the -acquire and -release + operations to build robust locking, but it can be set + on any key. The default value is empty (no session). +` + return strings.TrimSpace(helpText) +} + +func (c *KVPutCommand) Run(args []string) int { + cmdFlags := flag.NewFlagSet("get", flag.ContinueOnError) + cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } + httpAddr := HTTPAddrFlag(cmdFlags) + datacenter := cmdFlags.String("datacenter", "", "") + token := cmdFlags.String("token", "", "") + stale := cmdFlags.Bool("stale", false, "") + cas := cmdFlags.Bool("cas", false, "") + flags := cmdFlags.Uint64("flags", 0, "") + modifyIndex := cmdFlags.Uint64("modify-index", 0, "") + session := cmdFlags.String("session", "", "") + acquire := cmdFlags.Bool("acquire", false, "") + release := cmdFlags.Bool("release", false, "") + if err := cmdFlags.Parse(args); err != nil { + return 1 + } + + // Check for arg validation + args = cmdFlags.Args() + key, data, err := c.dataFromArgs(args) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! %s", err)) + return 1 + } + + if (*release || *acquire) && *session == "" { + c.Ui.Error("Error! Missing -session (required with -acquire and -release)") + return 1 + } + + // Create and test the HTTP client + conf := api.DefaultConfig() + conf.Address = *httpAddr + conf.Token = *token + client, err := api.NewClient(conf) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) + return 1 + } + + pair := &api.KVPair{ + Key: key, + ModifyIndex: *modifyIndex, + Flags: *flags, + Value: []byte(data), + Session: *session, + } + + wo := &api.WriteOptions{ + Datacenter: *datacenter, + Token: *token, + } + + switch { + case *cas: + // If the user did not supply a -modify-index, but wants a check-and-set, + // grab the current modify index and store that on the key. + if pair.ModifyIndex == 0 { + currentPair, _, err := client.KV().Get(key, &api.QueryOptions{ + Datacenter: *datacenter, + Token: *token, + AllowStale: *stale, + }) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! Could not get current key: %s", err)) + return 1 + } + if currentPair != nil { + pair.ModifyIndex = currentPair.ModifyIndex + } + } + + ok, _, err := client.KV().CAS(pair, wo) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! Did not write to %s: %s", key, err)) + return 1 + } + if !ok { + c.Ui.Error(fmt.Sprintf("Error! Did not write to %s: CAS failed", key)) + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Data written to: %s", key)) + return 0 + case *acquire: + ok, _, err := client.KV().Acquire(pair, wo) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! Failed writing data: %s", err)) + return 1 + } + if !ok { + c.Ui.Error("Error! Did not acquire lock") + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Lock acquired on: %s", key)) + return 0 + case *release: + ok, _, err := client.KV().Release(pair, wo) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! Failed writing data: %s", key)) + return 1 + } + if !ok { + c.Ui.Error("Error! Did not release lock") + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Lock released on: %s", key)) + return 0 + default: + if _, err := client.KV().Put(pair, wo); err != nil { + c.Ui.Error(fmt.Sprintf("Error! Failed writing data: %s", err)) + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Data written to: %s", key)) + return 0 + } +} + +func (c *KVPutCommand) Synopsis() string { + return "Sets or updates data in the KV store" +} + +func (c *KVPutCommand) dataFromArgs(args []string) (string, string, error) { + var stdin io.Reader = os.Stdin + if c.testStdin != nil { + stdin = c.testStdin + } + + switch len(args) { + case 0: + return "", "", fmt.Errorf("Missing KEY argument") + case 1: + return args[0], "", nil + case 2: + default: + return "", "", fmt.Errorf("Too many arguments (expected 1 or 2, got %d)", len(args)) + } + + key := args[0] + data := args[1] + + switch data[0] { + case '@': + data, err := ioutil.ReadFile(data[1:]) + if err != nil { + return "", "", fmt.Errorf("Failed to read file: %s", err) + } + return key, string(data), nil + case '-': + var b bytes.Buffer + if _, err := io.Copy(&b, stdin); err != nil { + return "", "", fmt.Errorf("Failed to read stdin: %s", err) + } + return key, b.String(), nil + default: + return key, data, nil + } +} diff --git a/command/kv_put_test.go b/command/kv_put_test.go new file mode 100644 index 0000000000..374a182d2b --- /dev/null +++ b/command/kv_put_test.go @@ -0,0 +1,309 @@ +package command + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strconv" + "strings" + "testing" + + "github.com/hashicorp/consul/api" + "github.com/mitchellh/cli" +) + +func TestKVPutCommand_implements(t *testing.T) { + var _ cli.Command = &KVPutCommand{} +} + +func TestKVPutCommand_noTabs(t *testing.T) { + assertNoTabs(t, new(KVPutCommand)) +} + +func TestKVPutCommand_Validation(t *testing.T) { + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + cases := map[string]struct { + args []string + output string + }{ + "-acquire without -session": { + []string{"-acquire", "foo"}, + "Missing -session", + }, + "-release without -session": { + []string{"-release", "foo"}, + "Missing -session", + }, + "no key": { + []string{}, + "Missing KEY argument", + }, + "extra args": { + []string{"foo", "bar", "baz"}, + "Too many arguments", + }, + } + + for name, tc := range cases { + // Ensure our buffer is always clear + if ui.ErrorWriter != nil { + ui.ErrorWriter.Reset() + } + if ui.OutputWriter != nil { + ui.OutputWriter.Reset() + } + + code := c.Run(tc.args) + if code == 0 { + t.Errorf("%s: expected non-zero exit", name) + } + + output := ui.ErrorWriter.String() + if !strings.Contains(output, tc.output) { + t.Errorf("%s: expected %q to contain %q", name, output, tc.output) + } + } +} + +func TestKVPutCommand_Run(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + args := []string{ + "-http-addr=" + srv.httpAddr, + "foo", "bar", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data.Value, []byte("bar")) { + t.Errorf("bad: %#v", data.Value) + } +} + +func TestKVPutCommand_File(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + f, err := ioutil.TempFile("", "kv-put-command-file") + if err != nil { + t.Fatalf("err: %#v", err) + } + defer os.Remove(f.Name()) + if _, err := f.WriteString("bar"); err != nil { + t.Fatalf("err: %#v", err) + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "foo", "@" + f.Name(), + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data.Value, []byte("bar")) { + t.Errorf("bad: %#v", data.Value) + } +} + +func TestKVPutCommand_FileNoExist(t *testing.T) { + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + args := []string{ + "foo", "@/nope/definitely/not-a-real-file.txt", + } + + code := c.Run(args) + if code == 0 { + t.Fatal("bad: expected error") + } + + output := ui.ErrorWriter.String() + if !strings.Contains(output, "Failed to read file") { + t.Errorf("bad: %#v", output) + } +} + +func TestKVPutCommand_Stdin(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + stdinR, stdinW := io.Pipe() + + ui := new(cli.MockUi) + c := &KVPutCommand{ + Ui: ui, + testStdin: stdinR, + } + + go func() { + stdinW.Write([]byte("bar")) + stdinW.Close() + }() + + args := []string{ + "-http-addr=" + srv.httpAddr, + "foo", "-", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data.Value, []byte("bar")) { + t.Errorf("bad: %#v", data.Value) + } +} + +func TestKVPutCommand_Flags(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-flags", "12345", + "foo", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + if data.Flags != 12345 { + t.Errorf("bad: %#v", data.Flags) + } +} + +func TestKVPutCommand_CAS(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-cas", + "foo", "a", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data.Value, []byte("a")) { + t.Errorf("bad: %#v", data.Value) + } +} + +func TestKVPutCommand_CASModifyIndex(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + // Create the initial pair so it has a ModifyIndex. + pair := &api.KVPair{ + Key: "foo", + Value: []byte("bar"), + } + if _, err := client.KV().Put(pair, nil); err != nil { + t.Fatalf("err: %#v", err) + } + + ui := new(cli.MockUi) + c := &KVPutCommand{Ui: ui} + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-cas", + "-modify-index", "123", + "foo", "a", + } + + code := c.Run(args) + if code == 0 { + t.Fatalf("bad: expected error") + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + // Reset buffers + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + args = []string{ + "-http-addr=" + srv.httpAddr, + "-cas", + "-modify-index", strconv.FormatUint(data.ModifyIndex, 10), + "foo", "a", + } + + code = c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err = client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data.Value, []byte("a")) { + t.Errorf("bad: %#v", data.Value) + } +} diff --git a/commands.go b/commands.go index 7b8c37775f..6875c5f26c 100644 --- a/commands.go +++ b/commands.go @@ -65,6 +65,12 @@ func init() { }, nil }, + "kv put": func() (cli.Command, error) { + return &command.KVPutCommand{ + Ui: ui, + }, nil + }, + "join": func() (cli.Command, error) { return &command.JoinCommand{ Ui: ui, diff --git a/website/source/docs/commands/kv/put.html.markdown.erb b/website/source/docs/commands/kv/put.html.markdown.erb new file mode 100644 index 0000000000..75098e7324 --- /dev/null +++ b/website/source/docs/commands/kv/put.html.markdown.erb @@ -0,0 +1,144 @@ +--- +layout: "docs" +page_title: "Commands: KV Put" +sidebar_current: "docs-commands-kv-put" +--- + +# Consul KV Put + +Command: `consul kv put` + +The `kv put` command writes the data to the given path in the key-value store. +The data can be of any type, but it will be transported as a base64-encoded +string for safe transport. The client will transparently handle the encoding +and decoding of these values. + +## Usage + +Usage: `consul kv put [options] KEY [DATA]` + +#### API Options + +<%= partial "docs/commands/http_api_options" %> + +#### KV Put Options + +* `-acquire` - Obtain a lock on the key. If the key does not exist, this + operation will create the key and obtain the lock. The session must already + exist and be specified via the -session flag. The default value is false. + +* `-cas` - Perform a Check-And-Set operation. If this value is specified without + -modify-index, the key will first be fetched and the resulting ModifyIndex + will be used on the next query. The default value is false. + +* `-flags=` - Unsigned integer value to assign to this key-value pair. This + value is not read by Consul, so clients can use this value however makes sense + for their use case. The default value is 0 (no flags). + +* `-modify-index=` - Unsigned integer representing the ModifyIndex of the + key. This is often combined with the -cas flag, but it can be specified for + any key. The default value is 0. + +* `-release` - Forfeit the lock on the key at the givne path. This requires the + -session flag to be set. The key must be held by the session in order to be + unlocked. The default value is false. + +* `-session=` - User-defined identifer for this session as a string. + This is commonly used with the -acquire and -release operations to build + robust locking, but it can be set on any key. The default value is empty (no + session). + +## Examples + +To insert a value of "5" for the key named "redis/config/connections" in the +key-value store: + +``` +$ consul kv put redis/config/connections 5 +Success! Data written to: redis/config/connections +``` + +If no data is specified, the key will be created with empty data: + +``` +$ consul kv put redis/config/connections +Success! Data written to: redis/config/connections +``` + +!> **Be careful when overwriting data!** The above operation would overwrite +the value at the key to the empty value. + +For longer or sensitive values, it is possible to read from a file by prefixing +with the `@` symbol: + +``` +$ consul kv put redis/config/password @password.txt +Success! Data written to: redis/config/connections +``` + +Or read values from stdin by specifying the `-` symbol: + +``` +$ echo "5" | consul kv put redis/config/password - +Success! Data written to: redis/config/connections + +$ consul kv put redis/config/password - +5 + +Success! Data written to: redis/config/connections +``` + +~> For secret and sensitive values, you should consider using a secret +management solution like **[HashiCorp's Vault](https://www.vaultproject.io/)**. +While it is possible to secure values in Consul's KV store, Vault provides a +more robust interface for secret management. + +To only update a key if it has not been modified since a given index, specify +the `-cas` and `-modify-index` flags: + +``` +$ consul kv get -detailed redis/config/connections | grep ModifyIndex +ModifyIndex 456 + +$ consul kv put -cas -modify-index=123 redis/config/connections 10 +Error! Did not write to redis/config/connections: CAS failed + +$ consul kv put -cas -modify-index=456 redis/config/connections 10 +Success! Data written to: redis/config/connections +``` + +It is also possible to have Consul fetch the current ModifyIndex before making +the query, by omitting the `-modify-index` flag. If the data is changed between +the initial read and the write, the operation will fail. + +``` +$ consul kv put -cas redis/config/connections 10 +Success! Data written to: redis/config/connections +``` + +To specify flags on the key, use the `-flags` option. These flags are completely +controlled by the user: + +``` +$ consul kv put -flags=42 redis/config/password s3cr3t +Success! Data written to: redis/config/password +``` + +To create or tune a lock, use the `-acquire` and `-session` flags. Note that the session must already exist (this command will not create it or manage it): + +``` +$ consul kv put -acquire -session=abc123 redis/lock/update +Success! Lock acquired on: redis/lock/update +``` + +When you are finished, release the lock: + +``` +$ consul kv put -release -session=acb123 redis/lock/update +Success! Lock released on: redis/lock/update +``` + +~> **Warning!** If you are trying to build a locking mechanism with these +low-level primitives, you may want to look at the [consul +lock](/docs/commands/lock.html) command. It provides higher-level +functionality without exposing the internal APIs of Consul. diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 2738d14b43..1668ad77fc 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -105,6 +105,9 @@ > get + > + put + From 297a22383f96c3bb770a39d407b2bfc2f126b676 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:12:40 -0500 Subject: [PATCH 07/22] Add kv delete command --- command/kv_delete.go | 178 ++++++++++++++++++ command/kv_delete_test.go | 143 ++++++++++++++ commands.go | 6 + .../docs/commands/kv/delete.html.markdown.erb | 96 ++++++++++ website/source/layouts/docs.erb | 3 + 5 files changed, 426 insertions(+) create mode 100644 command/kv_delete.go create mode 100644 command/kv_delete_test.go create mode 100644 website/source/docs/commands/kv/delete.html.markdown.erb diff --git a/command/kv_delete.go b/command/kv_delete.go new file mode 100644 index 0000000000..6857f1f209 --- /dev/null +++ b/command/kv_delete.go @@ -0,0 +1,178 @@ +package command + +import ( + "flag" + "fmt" + "strings" + + "github.com/hashicorp/consul/api" + "github.com/mitchellh/cli" +) + +// KVDeleteCommand is a Command implementation that is used to setup +// a "watch" which uses a sub-process +type KVDeleteCommand struct { + Ui cli.Ui +} + +func (c *KVDeleteCommand) Help() string { + helpText := ` +Usage: consul kv delete [options] KEY_OR_PREFIX + + Removes the value from Consul's key-value store at the given path. If no + key exists at the path, no action is taken. + + To delete the value for the key named "foo" in the key-value store: + + $ consul kv delete foo + + To delete all keys which start with "foo", specify the -recurse option: + + $ consul kv delete -recurse foo + + This will delete the keys named "foo", "food", and "foo/bar/zip" if they + existed. + +` + apiOptsText + ` + +KV Delete Options: + + -cas Perform a Check-And-Set operation. If this value is + specified without -modify-index, the key will first be + fetched and the resulting ModifyIndex will be used on + the next query. The default value is false. + + -modify-index= Unsigned integer representing the ModifyIndex of the + key. This is often combined with the -cas flag, but it + can be specified for any key. The default value is 0. + + -recurse Recursively delete all keys with the path. The default + value is false. +` + return strings.TrimSpace(helpText) +} + +func (c *KVDeleteCommand) Run(args []string) int { + cmdFlags := flag.NewFlagSet("get", flag.ContinueOnError) + cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } + datacenter := cmdFlags.String("datacenter", "", "") + token := cmdFlags.String("token", "", "") + stale := cmdFlags.Bool("stale", false, "") + cas := cmdFlags.Bool("cas", false, "") + modifyIndex := cmdFlags.Uint64("modify-index", 0, "") + recurse := cmdFlags.Bool("recurse", false, "") + httpAddr := HTTPAddrFlag(cmdFlags) + if err := cmdFlags.Parse(args); err != nil { + return 1 + } + + key := "" + + // Check for arg validation + args = cmdFlags.Args() + switch len(args) { + case 0: + key = "" + case 1: + key = args[0] + default: + c.Ui.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args))) + return 1 + } + + // This is just a "nice" thing to do. Since pairs cannot start with a /, but + // users will likely put "/" or "/foo", lets go ahead and strip that for them + // here. + if len(key) > 0 && key[0] == '/' { + key = key[1:] + } + + // If the key is empty and we are not doing a recursive delete, this is an + // error. + if key == "" && !*recurse { + c.Ui.Error("Error! Missing KEY argument") + return 1 + } + + // It is not valid to use a CAS and recurse in the same call + if *recurse && *cas { + c.Ui.Error("Cannot specify both -cas and -recurse!") + return 1 + } + if *recurse && *modifyIndex != 0 { + c.Ui.Error("Cannot specify both -modify-index and -recurse!") + return 1 + } + + // Create and test the HTTP client + conf := api.DefaultConfig() + conf.Address = *httpAddr + conf.Token = *token + client, err := api.NewClient(conf) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) + return 1 + } + + wo := &api.WriteOptions{ + Datacenter: *datacenter, + } + + switch { + case *recurse: + if _, err := client.KV().DeleteTree(key, wo); err != nil { + c.Ui.Error(fmt.Sprintf("Error! Did not delete prefix %s: %s", key, err)) + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Deleted keys with prefix: %s", key)) + return 0 + case *cas: + pair := &api.KVPair{ + Key: key, + ModifyIndex: *modifyIndex, + } + + // If the user did not supply a -modify-index, but wants a check-and-set, + // grab the current modify index and store that on the key. + if pair.ModifyIndex == 0 { + currentPair, _, err := client.KV().Get(key, &api.QueryOptions{ + Datacenter: *datacenter, + Token: *token, + AllowStale: *stale, + }) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! Could not get current key: %s", err)) + return 1 + } + if currentPair != nil { + pair.ModifyIndex = currentPair.ModifyIndex + } + } + + success, _, err := client.KV().DeleteCAS(pair, wo) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error! Did not delete key %s: %s", key, err)) + return 1 + } + if !success { + c.Ui.Error(fmt.Sprintf("Error! Did not delete key %s: CAS failed", key)) + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Deleted key: %s", key)) + return 0 + default: + if _, err := client.KV().Delete(key, wo); err != nil { + c.Ui.Error(fmt.Sprintf("Error deleting key %s: %s", key, err)) + return 1 + } + + c.Ui.Info(fmt.Sprintf("Success! Deleted key: %s", key)) + return 0 + } +} + +func (c *KVDeleteCommand) Synopsis() string { + return "Removes data from the KV store" +} diff --git a/command/kv_delete_test.go b/command/kv_delete_test.go new file mode 100644 index 0000000000..8768538ca2 --- /dev/null +++ b/command/kv_delete_test.go @@ -0,0 +1,143 @@ +package command + +import ( + "strings" + "testing" + + "github.com/hashicorp/consul/api" + "github.com/mitchellh/cli" +) + +func TestKVDeleteCommand_implements(t *testing.T) { + var _ cli.Command = &KVDeleteCommand{} +} + +func TestKVDeleteCommand_noTabs(t *testing.T) { + assertNoTabs(t, new(KVDeleteCommand)) +} + +func TestKVDeleteCommand_Validation(t *testing.T) { + ui := new(cli.MockUi) + c := &KVDeleteCommand{Ui: ui} + + cases := map[string]struct { + args []string + output string + }{ + "-cas and -recurse": { + []string{"-cas", "-recurse"}, + "Cannot specify both", + }, + "-modify-index and -recurse": { + []string{"-modify-index", "2", "-recurse"}, + "Cannot specify both", + }, + "no key": { + []string{}, + "Missing KEY argument", + }, + "extra args": { + []string{"foo", "bar", "baz"}, + "Too many arguments", + }, + } + + for name, tc := range cases { + // Ensure our buffer is always clear + if ui.ErrorWriter != nil { + ui.ErrorWriter.Reset() + } + if ui.OutputWriter != nil { + ui.OutputWriter.Reset() + } + + code := c.Run(tc.args) + if code == 0 { + t.Errorf("%s: expected non-zero exit", name) + } + + output := ui.ErrorWriter.String() + if !strings.Contains(output, tc.output) { + t.Errorf("%s: expected %q to contain %q", name, output, tc.output) + } + } +} + +func TestKVDeleteCommand_Run(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVDeleteCommand{Ui: ui} + + pair := &api.KVPair{ + Key: "foo", + Value: []byte("bar"), + } + _, err := client.KV().Put(pair, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "foo", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + pair, _, err = client.KV().Get("foo", nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + if pair != nil { + t.Fatalf("bad: %#v", pair) + } +} + +func TestKVDeleteCommand_Recurse(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVDeleteCommand{Ui: ui} + + keys := []string{"foo/a", "foo/b", "food"} + + for _, k := range keys { + pair := &api.KVPair{ + Key: k, + Value: []byte("bar"), + } + _, err := client.KV().Put(pair, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-recurse", + "foo", + } + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + for _, k := range keys { + pair, _, err := client.KV().Get(k, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + if pair != nil { + t.Fatalf("bad: %#v", pair) + } + } +} diff --git a/commands.go b/commands.go index 6875c5f26c..41f60b3c09 100644 --- a/commands.go +++ b/commands.go @@ -59,6 +59,12 @@ func init() { }, nil }, + "kv delete": func() (cli.Command, error) { + return &command.KVDeleteCommand{ + Ui: ui, + }, nil + }, + "kv get": func() (cli.Command, error) { return &command.KVGetCommand{ Ui: ui, diff --git a/website/source/docs/commands/kv/delete.html.markdown.erb b/website/source/docs/commands/kv/delete.html.markdown.erb new file mode 100644 index 0000000000..e9ec96590a --- /dev/null +++ b/website/source/docs/commands/kv/delete.html.markdown.erb @@ -0,0 +1,96 @@ +--- +layout: "docs" +page_title: "Commands: KV Delete" +sidebar_current: "docs-commands-kv-delete" +--- + +# Consul KV Delete + +Command: `consul kv delete` + +The `kv delete` command removes the value from Consul's key-value store at the +given path. If no key exists at the path, no action is taken. + +## Usage + +Usage: `consul kv delete [options] KEY_OR_PREFIX` + +#### API Options + +<%= partial "docs/commands/http_api_options" %> + +#### KV Delete Options + +* `-cas` - Perform a Check-And-Set operation. If this value is specified without + -modify-index, the key will first be fetched and the resulting ModifyIndex + will be used on the next query. The default value is false. + +* `-modify-index=` - Unsigned integer representing the ModifyIndex of the + key. This is often combined with the -cas flag, but it can be specified for + any key. The default value is 0. + +* `-recurse` - Recursively delete all keys with the path. The default value is + false. + +## Examples + +To remove the value for the key named "redis/config/connections" in the +key-value store: + +``` +$ consul kv delete redis/config/connections +Success! Deleted key: redis/config/connections +``` + +If the key does not exist, the command will not error, and a success message +will be returned: + +``` +$ consul kv delete not-a-real-key +Success! Deleted key: not-a-real-key +``` + +To only delete a key if it has not been modified since a given index, specify +the `-cas` and `-modify-index` flags: + +``` +$ consul kv get -detailed redis/config/connections | grep ModifyIndex +ModifyIndex 456 + +$ consul kv delete -cas -modify-index=123 redis/config/connections +Error! Did not delete key redis/config/connections: CAS failed + +$ consul kv delete -cas -modify-index=456 redis/config/connections +Success! Deleted key: redis/config/connections +``` + +It is also possible to have Consul fetch the current ModifyIndex before making +the query, by omitting the `-modify-index` flag. If the data is changed between +the initial read and the write, the operation will fail. + +``` +$ consul kv delete -cas redis/config/connections +Success! Deleted key: redis/config/connections +``` + +To recursively delete all keys that start with a given prefix, specify the +`-recurse` flag: + +``` +$ consul kv delete -recurse redis/ +Success! Deleted keys with prefix: redis/ +``` + +!> **Trailing slashes are important** in the recursive delete operation, since +Consul performs a greedy match on the provided prefix. If you were to use "foo" +as the key, this would recursively delete any key starting with those letters +such as "foo", "food", and "football" not just "foo". To ensure you are deleting +a folder, always use a trailing slash. + +It is not valid to combine the `-cas` option with `-recurse`, since you are +deleting multiple keys under a prefix in a single operation: + +``` +$ consul kv delete -cas -recurse redis/ +Cannot specify both -cas and -recurse! +``` diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 1668ad77fc..7c56891080 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -102,6 +102,9 @@ > kv
    + > + delete + > get From a6262c863b441c169eea67d038ce5faa25dc9b60 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:12:53 -0500 Subject: [PATCH 08/22] Add top-level docs command --- website/source/docs/commands/kv.html.markdown | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 website/source/docs/commands/kv.html.markdown diff --git a/website/source/docs/commands/kv.html.markdown b/website/source/docs/commands/kv.html.markdown new file mode 100644 index 0000000000..f44a49bc82 --- /dev/null +++ b/website/source/docs/commands/kv.html.markdown @@ -0,0 +1,82 @@ +--- +layout: "docs" +page_title: "Commands: KV" +sidebar_current: "docs-commands-kv" +--- + +# Consul KV + +Command: `consul kv` + +The `kv` command is used to interact with Consul's key-value store via the +command line. It exposes top-level commands for inserting, updating, reading, +and deleting from the store. + +The key-value store is also accessible via the +[HTTP API](docs/agent/http/kv.html). + +## Usage + +Usage: `consul kv ` + +For the exact documentation for your Consul version, run `consul kv -h` to view +the complete list of subcommands. + +```text +Usage: consul kv [options] [args] + + # ... + +Subcommands: + + delete Removes data from the KV store + get Retrieves or lists data from the KV store + put Sets or updates data in the KV store +``` + +For more information, examples, and usage about a subcommand, click on the name +of the subcommand in the sidebar or one of the links below: + +- [delete](/docs/commands/kv/delete.html) +- [get](/docs/commands/kv/get.html) +- [put](/docs/commands/kv/put.html) + +## Basic Examples + +To create or update the key named "redis/config/connections" to the value "5" in +Consul's key-value store: + +```text +$ consul kv put redis/config/connections 5 +Success! Data written to: redis/config/connections +``` + +To read a value back from Consul: + +```text +$ consul kv get redis/config/connections +5 +``` + +Or you can query for detailed information: + +```text +$ consul kv get -detailed redis/config/connections +CreateIndex 336 +Flags 0 +Key redis/config/connections +LockIndex 0 +ModifyIndex 336 +Session - +Value 5 +``` + +Finally, deleting a key is just as easy: + +```text +$ consul kv delete redis/config/connections +Success! Data deleted at key: redis/config/connections +``` + +For more examples, ask for subcommand help or view the subcommand documentation +by clicking on one of the links in the sidebar. From de7fc2c7acb359f3c485651e35034129346201ef Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:13:20 -0500 Subject: [PATCH 09/22] Fix styles --- website/source/assets/stylesheets/_docs.scss | 8 ++++++++ website/source/assets/stylesheets/_global.scss | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/website/source/assets/stylesheets/_docs.scss b/website/source/assets/stylesheets/_docs.scss index 5591a6d195..0a9d3a2d8f 100755 --- a/website/source/assets/stylesheets/_docs.scss +++ b/website/source/assets/stylesheets/_docs.scss @@ -171,6 +171,14 @@ body.layout-intro{ padding-left: 3%; padding-bottom: 80px; + .alert { + a { + color: inherit; + font-size: inherit; + font-weight: inherit; + } + } + .lead{ margin-bottom: 48px } diff --git a/website/source/assets/stylesheets/_global.scss b/website/source/assets/stylesheets/_global.scss index 7fb37ad7bf..bb34b2df39 100755 --- a/website/source/assets/stylesheets/_global.scss +++ b/website/source/assets/stylesheets/_global.scss @@ -51,6 +51,10 @@ pre { margin-bottom: 0; } +tt { + font-size: 18px; + font-family: "Menlo", "Monaco", "Courier New", monospace; +} //fixed grid below 992 to prevent smaller responsive sizes @media (max-width: 992px) { From 6cdf354270bb395b3c33daca6970c1c7480fa326 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 10:22:50 -0500 Subject: [PATCH 10/22] Fix copy-paste comments --- command/kv_delete.go | 4 ++-- command/kv_get.go | 4 ++-- command/kv_put.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/command/kv_delete.go b/command/kv_delete.go index 6857f1f209..197f62eaa2 100644 --- a/command/kv_delete.go +++ b/command/kv_delete.go @@ -9,8 +9,8 @@ import ( "github.com/mitchellh/cli" ) -// KVDeleteCommand is a Command implementation that is used to setup -// a "watch" which uses a sub-process +// KVDeleteCommand is a Command implementation that is used to delete a key or +// prefix of keys from the key-value store. type KVDeleteCommand struct { Ui cli.Ui } diff --git a/command/kv_get.go b/command/kv_get.go index 0358d7085f..1a54b37769 100644 --- a/command/kv_get.go +++ b/command/kv_get.go @@ -12,8 +12,8 @@ import ( "github.com/mitchellh/cli" ) -// KVGetCommand is a Command implementation that is used to setup -// a "watch" which uses a sub-process +// KVGetCommand is a Command implementation that is used to fetch the value of +// a key from the key-value store. type KVGetCommand struct { Ui cli.Ui } diff --git a/command/kv_put.go b/command/kv_put.go index 363571c3d6..b6f4117377 100644 --- a/command/kv_put.go +++ b/command/kv_put.go @@ -13,8 +13,8 @@ import ( "github.com/mitchellh/cli" ) -// KVPutCommand is a Command implementation that is used to setup -// a "watch" which uses a sub-process +// KVPutCommand is a Command implementation that is used to write data to the +// key-value store. type KVPutCommand struct { Ui cli.Ui From b3c280772fa1809b4684ea0819badfcfc11a4e10 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 11:24:39 -0500 Subject: [PATCH 11/22] Gemfile was wonky --- website/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/Gemfile.lock b/website/Gemfile.lock index 8925928e23..be13244148 100644 --- a/website/Gemfile.lock +++ b/website/Gemfile.lock @@ -124,7 +124,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) mini_portile2 (2.1.0) - minitest (5.9.0) + minitest (5.9.1) multi_json (1.12.1) nokogiri (1.6.8) mini_portile2 (~> 2.1.0) From 12e46503f77cf8f402329c61d1c3e65f3dd8dc34 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 13:31:19 -0700 Subject: [PATCH 12/22] Update ModifyIndex comment --- api/kv.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/kv.go b/api/kv.go index bbd49813c7..e1496bc8d7 100644 --- a/api/kv.go +++ b/api/kv.go @@ -19,7 +19,9 @@ type KVPair struct { // is a read-only field. CreateIndex uint64 - // ModifyIndex is used for the Check-And-Set operations. + // ModifyIndex is used for the Check-And-Set operations and can also be fed + // back into the WaitIndex of the QueryOptions in order to perform blocking + // queries. ModifyIndex uint64 // LockIndex holds the index corresponding to a lock on this key, if any. This From 55f1b4ac44b14481122f37ccdcb527ed59a4c69c Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 13:31:26 -0700 Subject: [PATCH 13/22] Rename session name to session ID --- api/kv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/kv.go b/api/kv.go index e1496bc8d7..e959937c0b 100644 --- a/api/kv.go +++ b/api/kv.go @@ -36,9 +36,9 @@ type KVPair struct { // base64 encoded upon transport. Value []byte - // Session is a string representing the name of the session. Any other + // Session is a string representing the ID of the session. Any othe // interactions with this key over the same session must specify the same - // session name. + // session ID. Session string } From c7496c565262c98e68cdee28af80491f5c2b0695 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:00:00 -0700 Subject: [PATCH 14/22] Change delete CAS behavior to require ModifyIndex --- command/kv_delete.go | 36 ++++------ command/kv_delete_test.go | 72 +++++++++++++++++-- .../docs/commands/kv/delete.html.markdown.erb | 12 +--- 3 files changed, 81 insertions(+), 39 deletions(-) diff --git a/command/kv_delete.go b/command/kv_delete.go index 197f62eaa2..4fa620a213 100644 --- a/command/kv_delete.go +++ b/command/kv_delete.go @@ -43,8 +43,7 @@ KV Delete Options: the next query. The default value is false. -modify-index= Unsigned integer representing the ModifyIndex of the - key. This is often combined with the -cas flag, but it - can be specified for any key. The default value is 0. + key. This is used in combination with the -cas flag. -recurse Recursively delete all keys with the path. The default value is false. @@ -57,7 +56,6 @@ func (c *KVDeleteCommand) Run(args []string) int { cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } datacenter := cmdFlags.String("datacenter", "", "") token := cmdFlags.String("token", "", "") - stale := cmdFlags.Bool("stale", false, "") cas := cmdFlags.Bool("cas", false, "") modifyIndex := cmdFlags.Uint64("modify-index", 0, "") recurse := cmdFlags.Bool("recurse", false, "") @@ -94,15 +92,22 @@ func (c *KVDeleteCommand) Run(args []string) int { return 1 } + // ModifyIndex is required for CAS + if *cas && *modifyIndex == 0 { + c.Ui.Error("Must specify -modify-index with -cas!") + return 1 + } + + // Specifying a ModifyIndex for a non-CAS operation is not possible. + if *modifyIndex != 0 && !*cas { + c.Ui.Error("Cannot specify -modify-index without -cas!") + } + // It is not valid to use a CAS and recurse in the same call if *recurse && *cas { c.Ui.Error("Cannot specify both -cas and -recurse!") return 1 } - if *recurse && *modifyIndex != 0 { - c.Ui.Error("Cannot specify both -modify-index and -recurse!") - return 1 - } // Create and test the HTTP client conf := api.DefaultConfig() @@ -133,23 +138,6 @@ func (c *KVDeleteCommand) Run(args []string) int { ModifyIndex: *modifyIndex, } - // If the user did not supply a -modify-index, but wants a check-and-set, - // grab the current modify index and store that on the key. - if pair.ModifyIndex == 0 { - currentPair, _, err := client.KV().Get(key, &api.QueryOptions{ - Datacenter: *datacenter, - Token: *token, - AllowStale: *stale, - }) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error! Could not get current key: %s", err)) - return 1 - } - if currentPair != nil { - pair.ModifyIndex = currentPair.ModifyIndex - } - } - success, _, err := client.KV().DeleteCAS(pair, wo) if err != nil { c.Ui.Error(fmt.Sprintf("Error! Did not delete key %s: %s", key, err)) diff --git a/command/kv_delete_test.go b/command/kv_delete_test.go index 8768538ca2..ac8572aadd 100644 --- a/command/kv_delete_test.go +++ b/command/kv_delete_test.go @@ -1,6 +1,7 @@ package command import ( + "strconv" "strings" "testing" @@ -25,12 +26,16 @@ func TestKVDeleteCommand_Validation(t *testing.T) { output string }{ "-cas and -recurse": { - []string{"-cas", "-recurse"}, + []string{"-cas", "-modify-index", "2", "-recurse", "foo"}, "Cannot specify both", }, - "-modify-index and -recurse": { - []string{"-modify-index", "2", "-recurse"}, - "Cannot specify both", + "-cas no -modify-index": { + []string{"-cas", "foo"}, + "Must specify -modify-index", + }, + "-modify-index no -cas": { + []string{"-modify-index", "2", "foo"}, + "Cannot specify -modify-index without", }, "no key": { []string{}, @@ -141,3 +146,62 @@ func TestKVDeleteCommand_Recurse(t *testing.T) { } } } + +func TestKVDeleteCommand_CAS(t *testing.T) { + srv, client := testAgentWithAPIClient(t) + defer srv.Shutdown() + waitForLeader(t, srv.httpAddr) + + ui := new(cli.MockUi) + c := &KVDeleteCommand{Ui: ui} + + pair := &api.KVPair{ + Key: "foo", + Value: []byte("bar"), + } + _, err := client.KV().Put(pair, nil) + if err != nil { + t.Fatalf("err: %#v", err) + } + + args := []string{ + "-http-addr=" + srv.httpAddr, + "-cas", + "-modify-index", "1", + "foo", + } + + code := c.Run(args) + if code == 0 { + t.Fatalf("bad: expected error") + } + + data, _, err := client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + + // Reset buffers + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + + args = []string{ + "-http-addr=" + srv.httpAddr, + "-cas", + "-modify-index", strconv.FormatUint(data.ModifyIndex, 10), + "foo", + } + + code = c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + data, _, err = client.KV().Get("foo", nil) + if err != nil { + t.Fatal(err) + } + if data != nil { + t.Fatalf("bad: %#v", data) + } +} diff --git a/website/source/docs/commands/kv/delete.html.markdown.erb b/website/source/docs/commands/kv/delete.html.markdown.erb index e9ec96590a..7d05bd7558 100644 --- a/website/source/docs/commands/kv/delete.html.markdown.erb +++ b/website/source/docs/commands/kv/delete.html.markdown.erb @@ -26,8 +26,7 @@ Usage: `consul kv delete [options] KEY_OR_PREFIX` will be used on the next query. The default value is false. * `-modify-index=` - Unsigned integer representing the ModifyIndex of the - key. This is often combined with the -cas flag, but it can be specified for - any key. The default value is 0. + key. This is used in combination with the -cas flag. * `-recurse` - Recursively delete all keys with the path. The default value is false. @@ -64,15 +63,6 @@ $ consul kv delete -cas -modify-index=456 redis/config/connections Success! Deleted key: redis/config/connections ``` -It is also possible to have Consul fetch the current ModifyIndex before making -the query, by omitting the `-modify-index` flag. If the data is changed between -the initial read and the write, the operation will fail. - -``` -$ consul kv delete -cas redis/config/connections -Success! Deleted key: redis/config/connections -``` - To recursively delete all keys that start with a given prefix, specify the `-recurse` flag: From 85350a432941c3cb487ec5978c40599f4589aad6 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:03:09 -0700 Subject: [PATCH 15/22] Update kv get docs --- website/source/docs/commands/kv/get.html.markdown.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/commands/kv/get.html.markdown.erb b/website/source/docs/commands/kv/get.html.markdown.erb index 29b36959da..4b316cb319 100644 --- a/website/source/docs/commands/kv/get.html.markdown.erb +++ b/website/source/docs/commands/kv/get.html.markdown.erb @@ -8,7 +8,7 @@ sidebar_current: "docs-commands-kv-get" Command: `consul kv get` -The `kv get` command is used to retrieves the value from Consul's key-value +The `kv get` command is used to retrieve the value from Consul's key-value store at the given key name. If no key exists with that name, an error is returned. If a key exists with that name but has no data, nothing is returned. If the name or prefix is omitted, it defaults to "" which is the root of the From 39fb70ff677f5be28e3422b684a2ae0d13527a9c Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:06:32 -0700 Subject: [PATCH 16/22] Fix CAS operations for put --- command/kv_put.go | 25 ++++---------- command/kv_put_test.go | 33 +++---------------- .../docs/commands/kv/put.html.markdown.erb | 16 ++------- 3 files changed, 14 insertions(+), 60 deletions(-) diff --git a/command/kv_put.go b/command/kv_put.go index b6f4117377..c67daa4a67 100644 --- a/command/kv_put.go +++ b/command/kv_put.go @@ -92,7 +92,6 @@ func (c *KVPutCommand) Run(args []string) int { httpAddr := HTTPAddrFlag(cmdFlags) datacenter := cmdFlags.String("datacenter", "", "") token := cmdFlags.String("token", "", "") - stale := cmdFlags.Bool("stale", false, "") cas := cmdFlags.Bool("cas", false, "") flags := cmdFlags.Uint64("flags", 0, "") modifyIndex := cmdFlags.Uint64("modify-index", 0, "") @@ -111,11 +110,18 @@ func (c *KVPutCommand) Run(args []string) int { return 1 } + // Session is reauired for release or acquire if (*release || *acquire) && *session == "" { c.Ui.Error("Error! Missing -session (required with -acquire and -release)") return 1 } + // ModifyIndex is required for CAS + if *cas && *modifyIndex == 0 { + c.Ui.Error("Must specify -modify-index with -cas!") + return 1 + } + // Create and test the HTTP client conf := api.DefaultConfig() conf.Address = *httpAddr @@ -141,23 +147,6 @@ func (c *KVPutCommand) Run(args []string) int { switch { case *cas: - // If the user did not supply a -modify-index, but wants a check-and-set, - // grab the current modify index and store that on the key. - if pair.ModifyIndex == 0 { - currentPair, _, err := client.KV().Get(key, &api.QueryOptions{ - Datacenter: *datacenter, - Token: *token, - AllowStale: *stale, - }) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error! Could not get current key: %s", err)) - return 1 - } - if currentPair != nil { - pair.ModifyIndex = currentPair.ModifyIndex - } - } - ok, _, err := client.KV().CAS(pair, wo) if err != nil { c.Ui.Error(fmt.Sprintf("Error! Did not write to %s: %s", key, err)) diff --git a/command/kv_put_test.go b/command/kv_put_test.go index 374a182d2b..8239809cc8 100644 --- a/command/kv_put_test.go +++ b/command/kv_put_test.go @@ -37,6 +37,10 @@ func TestKVPutCommand_Validation(t *testing.T) { []string{"-release", "foo"}, "Missing -session", }, + "-cas no -modify-index": { + []string{"-cas", "foo"}, + "Must specify -modify-index", + }, "no key": { []string{}, "Missing KEY argument", @@ -224,35 +228,6 @@ func TestKVPutCommand_CAS(t *testing.T) { defer srv.Shutdown() waitForLeader(t, srv.httpAddr) - ui := new(cli.MockUi) - c := &KVPutCommand{Ui: ui} - - args := []string{ - "-http-addr=" + srv.httpAddr, - "-cas", - "foo", "a", - } - - code := c.Run(args) - if code != 0 { - t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) - } - - data, _, err := client.KV().Get("foo", nil) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(data.Value, []byte("a")) { - t.Errorf("bad: %#v", data.Value) - } -} - -func TestKVPutCommand_CASModifyIndex(t *testing.T) { - srv, client := testAgentWithAPIClient(t) - defer srv.Shutdown() - waitForLeader(t, srv.httpAddr) - // Create the initial pair so it has a ModifyIndex. pair := &api.KVPair{ Key: "foo", diff --git a/website/source/docs/commands/kv/put.html.markdown.erb b/website/source/docs/commands/kv/put.html.markdown.erb index 75098e7324..cf2114b546 100644 --- a/website/source/docs/commands/kv/put.html.markdown.erb +++ b/website/source/docs/commands/kv/put.html.markdown.erb @@ -10,8 +10,8 @@ Command: `consul kv put` The `kv put` command writes the data to the given path in the key-value store. The data can be of any type, but it will be transported as a base64-encoded -string for safe transport. The client will transparently handle the encoding -and decoding of these values. +string for safe transport during reads. The client will transparently handle the +encoding and decoding of these values. ## Usage @@ -36,8 +36,7 @@ Usage: `consul kv put [options] KEY [DATA]` for their use case. The default value is 0 (no flags). * `-modify-index=` - Unsigned integer representing the ModifyIndex of the - key. This is often combined with the -cas flag, but it can be specified for - any key. The default value is 0. + key. This is used in combination with the -cas flag. * `-release` - Forfeit the lock on the key at the givne path. This requires the -session flag to be set. The key must be held by the session in order to be @@ -107,15 +106,6 @@ $ consul kv put -cas -modify-index=456 redis/config/connections 10 Success! Data written to: redis/config/connections ``` -It is also possible to have Consul fetch the current ModifyIndex before making -the query, by omitting the `-modify-index` flag. If the data is changed between -the initial read and the write, the operation will fail. - -``` -$ consul kv put -cas redis/config/connections 10 -Success! Data written to: redis/config/connections -``` - To specify flags on the key, use the `-flags` option. These flags are completely controlled by the user: From 40de5858d330190d3c204bb3e00bb4eaa67db799 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:06:44 -0700 Subject: [PATCH 17/22] Fix typo --- api/kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/kv.go b/api/kv.go index e959937c0b..2b93fa8539 100644 --- a/api/kv.go +++ b/api/kv.go @@ -36,7 +36,7 @@ type KVPair struct { // base64 encoded upon transport. Value []byte - // Session is a string representing the ID of the session. Any othe + // Session is a string representing the ID of the session. Any other // interactions with this key over the same session must specify the same // session ID. Session string From f9a7be9315a3fb76920e97ea515485aca56062fa Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:09:35 -0700 Subject: [PATCH 18/22] Remove debugging code --- command/kv_get_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/kv_get_test.go b/command/kv_get_test.go index e0dc8ce31a..815b4d72d1 100644 --- a/command/kv_get_test.go +++ b/command/kv_get_test.go @@ -246,7 +246,7 @@ func TestKVGetCommand_Recurse(t *testing.T) { output := ui.OutputWriter.String() for key, value := range keys { if !strings.Contains(output, key+":"+value) { - t.Fatalf("bad %#v missing %q", output, key+"adafa") + t.Fatalf("bad %#v missing %q", output, key) } } } From 28de64c7f0f7d2427ccab69dc76899aeefec2631 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:10:22 -0700 Subject: [PATCH 19/22] Remove base64 comment entirely --- website/source/docs/commands/kv/put.html.markdown.erb | 3 --- 1 file changed, 3 deletions(-) diff --git a/website/source/docs/commands/kv/put.html.markdown.erb b/website/source/docs/commands/kv/put.html.markdown.erb index cf2114b546..b8d070818c 100644 --- a/website/source/docs/commands/kv/put.html.markdown.erb +++ b/website/source/docs/commands/kv/put.html.markdown.erb @@ -9,9 +9,6 @@ sidebar_current: "docs-commands-kv-put" Command: `consul kv put` The `kv put` command writes the data to the given path in the key-value store. -The data can be of any type, but it will be transported as a base64-encoded -string for safe transport during reads. The client will transparently handle the -encoding and decoding of these values. ## Usage From d0ac899f79d814e0c8d22f123755e4d4e4a87a18 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:11:23 -0700 Subject: [PATCH 20/22] Fix up kv put help output --- command/kv_put.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/command/kv_put.go b/command/kv_put.go index c67daa4a67..f90306a332 100644 --- a/command/kv_put.go +++ b/command/kv_put.go @@ -27,8 +27,7 @@ func (c *KVPutCommand) Help() string { Usage: consul kv put [options] KEY [DATA] Writes the data to the given path in the key-value store. The data can be of - any type, but it will be transported as a base64-encoded string for safe - transport: + any type. $ consul kv put config/redis/maxconns 5 @@ -37,6 +36,10 @@ Usage: consul kv put [options] KEY [DATA] $ consul kv put config/program/license @license.lic + Or it can be read from stdin using the "-" symbol: + + $ echo "abcd1234" | consul kv put config/program/license - + The DATA argument itself is optional. If omitted, this will create an empty key-value pair at the specified path: From 1898d071ced83485b05a0e4a01a338839a041562 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:13:39 -0700 Subject: [PATCH 21/22] Fix kv put flags --- command/kv_delete.go | 7 +++---- command/kv_put.go | 10 ++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/command/kv_delete.go b/command/kv_delete.go index 4fa620a213..85b30b4c8f 100644 --- a/command/kv_delete.go +++ b/command/kv_delete.go @@ -37,10 +37,9 @@ Usage: consul kv delete [options] KEY_OR_PREFIX KV Delete Options: - -cas Perform a Check-And-Set operation. If this value is - specified without -modify-index, the key will first be - fetched and the resulting ModifyIndex will be used on - the next query. The default value is false. + -cas Perform a Check-And-Set operation. Specifying this + value also requires the -modify-index flag to be set. + The default value is false. -modify-index= Unsigned integer representing the ModifyIndex of the key. This is used in combination with the -cas flag. diff --git a/command/kv_put.go b/command/kv_put.go index f90306a332..28c5d98fa1 100644 --- a/command/kv_put.go +++ b/command/kv_put.go @@ -62,10 +62,9 @@ KV Put Options: lock. The session must already exist and be specified via the -session flag. The default value is false. - -cas Perform a Check-And-Set operation. If this value is - specified without -modify-index, the key will first be - fetched and the resulting ModifyIndex will be used on - the next query. The default value is false. + -cas Perform a Check-And-Set operation. Specifying this + value also requires the -modify-index flag to be set. + The default value is false. -flags= Unsigned integer value to assign to this key-value pair. This value is not read by Consul, so clients can @@ -73,8 +72,7 @@ KV Put Options: The default value is 0 (no flags). -modify-index= Unsigned integer representing the ModifyIndex of the - key. This is often combined with the -cas flag, but it - can be specified for any key. The default value is 0. + key. This is used in combination with the -cas flag. -release Forfeit the lock on the key at the givne path. This requires the -session flag to be set. The key must be From f381d19221e52ea7b25c00577ffa89685417aa13 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 26 Sep 2016 16:15:27 -0700 Subject: [PATCH 22/22] Update cas docs --- website/source/docs/commands/kv/delete.html.markdown.erb | 5 ++--- website/source/docs/commands/kv/put.html.markdown.erb | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/website/source/docs/commands/kv/delete.html.markdown.erb b/website/source/docs/commands/kv/delete.html.markdown.erb index 7d05bd7558..e716437cb7 100644 --- a/website/source/docs/commands/kv/delete.html.markdown.erb +++ b/website/source/docs/commands/kv/delete.html.markdown.erb @@ -21,9 +21,8 @@ Usage: `consul kv delete [options] KEY_OR_PREFIX` #### KV Delete Options -* `-cas` - Perform a Check-And-Set operation. If this value is specified without - -modify-index, the key will first be fetched and the resulting ModifyIndex - will be used on the next query. The default value is false. +* `-cas` - Perform a Check-And-Set operation. Specifying this value also + requires the -modify-index flag to be set. The default value is false. * `-modify-index=` - Unsigned integer representing the ModifyIndex of the key. This is used in combination with the -cas flag. diff --git a/website/source/docs/commands/kv/put.html.markdown.erb b/website/source/docs/commands/kv/put.html.markdown.erb index b8d070818c..eaf4c3f24c 100644 --- a/website/source/docs/commands/kv/put.html.markdown.erb +++ b/website/source/docs/commands/kv/put.html.markdown.erb @@ -24,9 +24,8 @@ Usage: `consul kv put [options] KEY [DATA]` operation will create the key and obtain the lock. The session must already exist and be specified via the -session flag. The default value is false. -* `-cas` - Perform a Check-And-Set operation. If this value is specified without - -modify-index, the key will first be fetched and the resulting ModifyIndex - will be used on the next query. The default value is false. +* `-cas` - Perform a Check-And-Set operation. Specifying this value also + requires the -modify-index flag to be set. The default value is false. * `-flags=` - Unsigned integer value to assign to this key-value pair. This value is not read by Consul, so clients can use this value however makes sense