From 10f325dc817afc593063d43c85d29b05910d0817 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 3 Nov 2020 18:00:44 -0600 Subject: [PATCH 01/27] Initial stab at snapshot inspect key breakdown --- command/snapshot/inspect/formatter.go | 25 ++++++- command/snapshot/inspect/snapshot_inspect.go | 73 +++++++++++++++++--- 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index 9b9a4c4c30..9f059a5dc3 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -15,7 +15,7 @@ const ( ) type Formatter interface { - Format(*OutputFormat) (string, error) + Format(*OutputFormat, *OutputFormat, bool) (string, error) } func GetSupportedFormats() []string { @@ -38,7 +38,7 @@ func NewFormatter(format string) (Formatter, error) { } } -func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) { +func (_ *prettyFormatter) Format(info *OutputFormat, kvInfo *OutputFormat, detailed bool) (string, error) { var b bytes.Buffer tw := tabwriter.NewWriter(&b, 8, 8, 6, ' ', 0) @@ -60,6 +60,25 @@ func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) { if err := tw.Flush(); err != nil { return b.String(), err } + + if detailed { + kvtw := tabwriter.NewWriter(&b, 30, 8, 12, ' ', 0) + + fmt.Fprintf(kvtw, "\n") + fmt.Fprintln(kvtw, "\n Key Name\tCount\tSize\t") + fmt.Fprintf(kvtw, " %s\t%s\t%s\t", "----", "----", "----") + // For each different type generate new output + for _, s := range kvInfo.Stats { + fmt.Fprintf(kvtw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) + } + fmt.Fprintf(kvtw, "\n %s\t%s\t%s\t", "----", "----", "----") + fmt.Fprintf(kvtw, "\n Total\t\t%s\t", ByteSize(uint64(kvInfo.TotalSize))) + + if err := kvtw.Flush(); err != nil { + return b.String(), err + } + } + return b.String(), nil } @@ -69,7 +88,7 @@ func newJSONFormatter() Formatter { return &jsonFormatter{} } -func (_ *jsonFormatter) Format(info *OutputFormat) (string, error) { +func (_ *jsonFormatter) Format(info *OutputFormat, infoKV *OutputFormat, detailed bool) (string, error) { b, err := json.MarshalIndent(info, "", " ") if err != nil { return "", fmt.Errorf("Failed to marshal original snapshot stats: %v", err) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 2e5b41e595..4cc894944d 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -29,10 +29,18 @@ type cmd struct { flags *flag.FlagSet help string format string + + // flags + detailed bool + kvDepth int } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) + c.flags.BoolVar(&c.detailed, "detailed", false, + "Provides detailed information about KV store data.") + c.flags.IntVar(&c.kvDepth, "kv-depth", 2, + "The key prefix depth used to breakdown KV store data. Defaults to 2.") c.flags.StringVar( &c.format, "format", @@ -57,6 +65,7 @@ type MetadataInfo struct { type OutputFormat struct { Meta *MetadataInfo Stats []typeStats + KStats []typeStats TotalSize int } @@ -101,7 +110,7 @@ func (c *cmd) Run(args []string) int { } }() - stats, totalSize, err := enhance(readFile) + stats, kstats, totalSize, err := enhance(readFile, c.detailed, c.kvDepth) if err != nil { c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err)) return 1 @@ -122,14 +131,20 @@ func (c *cmd) Run(args []string) int { } //Restructures stats given above to be human readable - formattedStats := generatetypeStats(stats) + formattedStats, formattedKStats := generatetypeStats(stats, kstats, c.detailed) in := &OutputFormat{ Meta: metaformat, Stats: formattedStats, TotalSize: totalSize, } - out, err := formatter.Format(in) + inKV := &OutputFormat{ + Meta: metaformat, + Stats: formattedKStats, + TotalSize: totalSize, + } + + out, err := formatter.Format(in, inKV, c.detailed) if err != nil { c.UI.Error(err.Error()) return 1 @@ -145,7 +160,7 @@ type typeStats struct { Count int } -func generatetypeStats(info map[structs.MessageType]typeStats) []typeStats { +func generatetypeStats(info map[structs.MessageType]typeStats, kvInfo map[string]typeStats, detailed bool) ([]typeStats, []typeStats) { ss := make([]typeStats, 0, len(info)) for _, s := range info { @@ -155,7 +170,20 @@ func generatetypeStats(info map[structs.MessageType]typeStats) []typeStats { // Sort the stat slice sort.Slice(ss, func(i, j int) bool { return ss[i].Sum > ss[j].Sum }) - return ss + if detailed { + ks := make([]typeStats, 0, len(kvInfo)) + + for _, s := range kvInfo { + ks = append(ks, s) + } + + // Sort the kv stat slice + sort.Slice(ks, func(i, j int) bool { return ks[i].Sum > ks[j].Sum }) + + return ss, ks + } + + return ss, nil } // countingReader helps keep track of the bytes we have read @@ -175,8 +203,9 @@ func (r *countingReader) Read(p []byte) (n int, err error) { // enhance utilizes ReadSnapshot to populate the struct with // all of the snapshot's itemized data -func enhance(file io.Reader) (map[structs.MessageType]typeStats, int, error) { +func enhance(file io.Reader, detailed bool, kvDepth int) (map[structs.MessageType]typeStats, map[string]typeStats, int, error) { stats := make(map[structs.MessageType]typeStats) + kstats := make(map[string]typeStats) cr := &countingReader{wrappedReader: file} totalSize := 0 handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error { @@ -185,6 +214,7 @@ func enhance(file io.Reader) (map[structs.MessageType]typeStats, int, error) { if s.Name == "" { s.Name = name } + var val interface{} err := dec.Decode(&val) if err != nil { @@ -196,12 +226,39 @@ func enhance(file io.Reader) (map[structs.MessageType]typeStats, int, error) { s.Count++ totalSize = cr.read stats[msg] = s + + if detailed { + if s.Name == "KVS" { + switch val := val.(type) { + case map[string]interface{}: + fmt.Println("map-match") + for k, v := range val { + depth := kvDepth + if k == "Key" { + split := strings.Split(v.(string), "/") + if depth > len(split) { + depth = len(split) + } + prefix := strings.Join(split[0:depth], "/") + kvs := kstats[prefix] + if kvs.Name == "" { + kvs.Name = prefix + } + kvs.Sum += size + kvs.Count++ + kstats[prefix] = kvs + } + } + } + } + } + return nil } if err := fsm.ReadSnapshot(cr, handler); err != nil { - return nil, 0, err + return nil, nil, 0, err } - return stats, totalSize, nil + return stats, kstats, totalSize, nil } From 262fb3a0c382ff5434d2787dbbce007942532b47 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 4 Nov 2020 10:11:20 -0600 Subject: [PATCH 02/27] Add snapshot inspect filter param --- command/snapshot/inspect/snapshot_inspect.go | 32 ++++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 4cc894944d..f5ff38e3d4 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -32,15 +32,18 @@ type cmd struct { // flags detailed bool - kvDepth int + depth int + filter string } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) c.flags.BoolVar(&c.detailed, "detailed", false, "Provides detailed information about KV store data.") - c.flags.IntVar(&c.kvDepth, "kv-depth", 2, + c.flags.IntVar(&c.depth, "depth", 2, "The key prefix depth used to breakdown KV store data. Defaults to 2.") + c.flags.StringVar(&c.filter, "filter", "", + "Filter KV keys using this prefix filter.") c.flags.StringVar( &c.format, "format", @@ -110,7 +113,7 @@ func (c *cmd) Run(args []string) int { } }() - stats, kstats, totalSize, err := enhance(readFile, c.detailed, c.kvDepth) + stats, kstats, totalSize, err := enhance(readFile, c.detailed, c.depth, c.filter) if err != nil { c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err)) return 1 @@ -203,7 +206,7 @@ func (r *countingReader) Read(p []byte) (n int, err error) { // enhance utilizes ReadSnapshot to populate the struct with // all of the snapshot's itemized data -func enhance(file io.Reader, detailed bool, kvDepth int) (map[structs.MessageType]typeStats, map[string]typeStats, int, error) { +func enhance(file io.Reader, detailed bool, depth int, filter string) (map[structs.MessageType]typeStats, map[string]typeStats, int, error) { stats := make(map[structs.MessageType]typeStats) kstats := make(map[string]typeStats) cr := &countingReader{wrappedReader: file} @@ -231,19 +234,28 @@ func enhance(file io.Reader, detailed bool, kvDepth int) (map[structs.MessageTyp if s.Name == "KVS" { switch val := val.(type) { case map[string]interface{}: - fmt.Println("map-match") for k, v := range val { - depth := kvDepth if k == "Key" { - split := strings.Split(v.(string), "/") - if depth > len(split) { - depth = len(split) + // check for whether a filter is specified. if it is, skip + // any keys that don't match. + if len(filter) > 0 && !strings.HasPrefix(v.(string), filter) { + break } - prefix := strings.Join(split[0:depth], "/") + + split := strings.Split(v.(string), "/") + + // handle the situation where the key is shorter than + // the specified depth. + actualDepth := depth + if depth > len(split) { + actualDepth = len(split) + } + prefix := strings.Join(split[0:actualDepth], "/") kvs := kstats[prefix] if kvs.Name == "" { kvs.Name = prefix } + kvs.Sum += size kvs.Count++ kstats[prefix] = kvs From e7a88d4a30e592d8b14d9990bfc4deaad5bdb7c9 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 4 Nov 2020 10:30:38 -0600 Subject: [PATCH 03/27] This ended up not being used. --- command/snapshot/inspect/snapshot_inspect.go | 1 - 1 file changed, 1 deletion(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index f5ff38e3d4..16087e0e25 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -68,7 +68,6 @@ type MetadataInfo struct { type OutputFormat struct { Meta *MetadataInfo Stats []typeStats - KStats []typeStats TotalSize int } From 208d6ebd9507920a99c172a8c1df35dd2da36bf5 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 4 Nov 2020 14:04:17 -0600 Subject: [PATCH 04/27] Get JSON formatting working --- command/snapshot/inspect/formatter.go | 10 +++++----- command/snapshot/inspect/snapshot_inspect.go | 9 +++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index 9f059a5dc3..89a3c368f2 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -15,7 +15,7 @@ const ( ) type Formatter interface { - Format(*OutputFormat, *OutputFormat, bool) (string, error) + Format(*OutputFormat, bool) (string, error) } func GetSupportedFormats() []string { @@ -38,7 +38,7 @@ func NewFormatter(format string) (Formatter, error) { } } -func (_ *prettyFormatter) Format(info *OutputFormat, kvInfo *OutputFormat, detailed bool) (string, error) { +func (_ *prettyFormatter) Format(info *OutputFormat, detailed bool) (string, error) { var b bytes.Buffer tw := tabwriter.NewWriter(&b, 8, 8, 6, ' ', 0) @@ -68,11 +68,11 @@ func (_ *prettyFormatter) Format(info *OutputFormat, kvInfo *OutputFormat, detai fmt.Fprintln(kvtw, "\n Key Name\tCount\tSize\t") fmt.Fprintf(kvtw, " %s\t%s\t%s\t", "----", "----", "----") // For each different type generate new output - for _, s := range kvInfo.Stats { + for _, s := range info.KStats { fmt.Fprintf(kvtw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) } fmt.Fprintf(kvtw, "\n %s\t%s\t%s\t", "----", "----", "----") - fmt.Fprintf(kvtw, "\n Total\t\t%s\t", ByteSize(uint64(kvInfo.TotalSize))) + fmt.Fprintf(kvtw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) if err := kvtw.Flush(); err != nil { return b.String(), err @@ -88,7 +88,7 @@ func newJSONFormatter() Formatter { return &jsonFormatter{} } -func (_ *jsonFormatter) Format(info *OutputFormat, infoKV *OutputFormat, detailed bool) (string, error) { +func (_ *jsonFormatter) Format(info *OutputFormat, detailed bool) (string, error) { b, err := json.MarshalIndent(info, "", " ") if err != nil { return "", fmt.Errorf("Failed to marshal original snapshot stats: %v", err) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 16087e0e25..2eaaa9d704 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -68,6 +68,7 @@ type MetadataInfo struct { type OutputFormat struct { Meta *MetadataInfo Stats []typeStats + KStats []typeStats TotalSize int } @@ -138,15 +139,11 @@ func (c *cmd) Run(args []string) int { in := &OutputFormat{ Meta: metaformat, Stats: formattedStats, - TotalSize: totalSize, - } - inKV := &OutputFormat{ - Meta: metaformat, - Stats: formattedKStats, + KStats: formattedKStats, TotalSize: totalSize, } - out, err := formatter.Format(in, inKV, c.detailed) + out, err := formatter.Format(in, c.detailed) if err != nil { c.UI.Error(err.Error()) return 1 From 4bf70dd857eab01dbca31454833dcfed3f28ddbb Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 4 Nov 2020 14:36:42 -0600 Subject: [PATCH 05/27] Cleanup formatter --- command/snapshot/inspect/formatter.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index 89a3c368f2..8dd02f6120 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -57,26 +57,20 @@ func (_ *prettyFormatter) Format(info *OutputFormat, detailed bool) (string, err fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) - if err := tw.Flush(); err != nil { - return b.String(), err - } - if detailed { - kvtw := tabwriter.NewWriter(&b, 30, 8, 12, ' ', 0) - - fmt.Fprintf(kvtw, "\n") - fmt.Fprintln(kvtw, "\n Key Name\tCount\tSize\t") - fmt.Fprintf(kvtw, " %s\t%s\t%s\t", "----", "----", "----") + fmt.Fprintf(tw, "\n") + fmt.Fprintln(tw, "\n Key Name\tCount\tSize\t") + fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") // For each different type generate new output for _, s := range info.KStats { - fmt.Fprintf(kvtw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) + fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) } - fmt.Fprintf(kvtw, "\n %s\t%s\t%s\t", "----", "----", "----") - fmt.Fprintf(kvtw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) + fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") + fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) + } - if err := kvtw.Flush(); err != nil { - return b.String(), err - } + if err := tw.Flush(); err != nil { + return b.String(), err } return b.String(), nil From 52ea53f95ea138e49a9ccdb46818f0877c26aaf0 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 10:31:36 -0600 Subject: [PATCH 06/27] Update docs with new flags --- website/pages/commands/snapshot/inspect.mdx | 136 +++++++++----------- 1 file changed, 61 insertions(+), 75 deletions(-) diff --git a/website/pages/commands/snapshot/inspect.mdx b/website/pages/commands/snapshot/inspect.mdx index fcf59223ee..0b039f6af6 100644 --- a/website/pages/commands/snapshot/inspect.mdx +++ b/website/pages/commands/snapshot/inspect.mdx @@ -38,87 +38,70 @@ To inspect a snapshot from the file "backup.snap": ```shell-session $ consul snapshot inspect backup.snap - ID 2-13-1603221729747 - Size 5141 - Index 13 + ID 2-12426-1604593650375 + Size 17228 + Index 12426 Term 2 Version 1 - Type Count Size - ---- ---- ---- - Register 3 1.7KB - ConnectCA 1 1.2KB - ConnectCAProviderState 1 1.1KB - Index 12 344B - Autopilot 1 199B - ConnectCAConfig 1 197B - FederationState 1 139B - SystemMetadata 1 68B - ChunkingState 1 12B - ---- ---- ---- - Total 5KB + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB ``` -To enhance a snapshot inespection from "backup.snap": +To get more details for a snapshot inspection from "backup.snap": + ```shell-session -$ consul snapshot inspect -format=json backup.snap -{ - "Meta": { - "ID": "2-13-1603221729747", - "Size": 5141, - "Index": 13, - "Term": 2, - "Version": 1 - }, - "Stats": [ - { - "Name": "Register", - "Sum": 1750, - "Count": 3 - }, - { - "Name": "ConnectCA", - "Sum": 1258, - "Count": 1 - }, - { - "Name": "ConnectCAProviderState", - "Sum": 1174, - "Count": 1 - }, - { - "Name": "Index", - "Sum": 344, - "Count": 12 - }, - { - "Name": "Autopilot", - "Sum": 199, - "Count": 1 - }, - { - "Name": "ConnectCAConfig", - "Sum": 197, - "Count": 1 - }, - { - "Name": "FederationState", - "Sum": 139, - "Count": 1 - }, - { - "Name": "SystemMetadata", - "Sum": 68, - "Count": 1 - }, - { - "Name": "ChunkingState", - "Sum": 12, - "Count": 1 - } - ], - "TotalSize": 5141 -} +$ consul snapshot inspect -detailed -depth 3 -filter vault/core backup.snap + ID 2-12426-1604593650375 + Size 17228 + Index 12426 + Term 2 + Version 1 + + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB + + Key Name Count Size + ---- ---- ---- + vault/core/leader 1 1.6KB + vault/core/mounts 1 675B + vault/core/wrapping 1 633B + vault/core/local-mounts 1 450B + vault/core/auth 1 423B + vault/core/cluster 2 388B + vault/core/keyring 1 320B + vault/core/master 1 237B + vault/core/seal-config 1 211B + vault/core/hsm 1 189B + vault/core/local-audit 1 185B + vault/core/local-auth 1 183B + vault/core/audit 1 179B + vault/core/lock 1 170B + vault/core/shamir-kek 1 159B + ---- ---- ---- + Total 5.9KB ``` Please see the [HTTP API](/api/snapshot) documentation for @@ -126,4 +109,7 @@ more details about snapshot internals. #### Command Options +- `-detailed` - Optional, provides a key space usage breakdown for any KV data stored in Consul. +- `-depth` - Optional, used with `-detailed` to adjust the grouping level of keys. Defaults to 2. +- `-filter` - Optional, used with `-detailed` to specify a key prefix that excludes keys that don't match. - `-format` - Optional, allows from changing the output to JSON. Parameters accepted are "pretty" and "JSON". From eafc593d0fa9d8acfb9f9478aae39fde3608e08a Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 10:32:23 -0600 Subject: [PATCH 07/27] Make key breakdown total size accurate --- command/snapshot/inspect/formatter.go | 4 +-- command/snapshot/inspect/snapshot_inspect.go | 36 +++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index 8dd02f6120..0531623f5d 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -62,11 +62,11 @@ func (_ *prettyFormatter) Format(info *OutputFormat, detailed bool) (string, err fmt.Fprintln(tw, "\n Key Name\tCount\tSize\t") fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") // For each different type generate new output - for _, s := range info.KStats { + for _, s := range info.StatsKV { fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) } fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") - fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) + fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSizeKV))) } if err := tw.Flush(); err != nil { diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 2eaaa9d704..1c1938dbbc 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -66,10 +66,11 @@ type MetadataInfo struct { // OutputFormat is used for passing information // through the formatter type OutputFormat struct { - Meta *MetadataInfo - Stats []typeStats - KStats []typeStats - TotalSize int + Meta *MetadataInfo + Stats []typeStats + StatsKV []typeStats + TotalSize int + TotalSizeKV int } func (c *cmd) Run(args []string) int { @@ -113,7 +114,7 @@ func (c *cmd) Run(args []string) int { } }() - stats, kstats, totalSize, err := enhance(readFile, c.detailed, c.depth, c.filter) + stats, statsKV, totalSize, totalSizeKV, err := enhance(readFile, c.detailed, c.depth, c.filter) if err != nil { c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err)) return 1 @@ -134,13 +135,14 @@ func (c *cmd) Run(args []string) int { } //Restructures stats given above to be human readable - formattedStats, formattedKStats := generatetypeStats(stats, kstats, c.detailed) + formattedStats, formattedStatsKV := generatetypeStats(stats, statsKV, c.detailed) in := &OutputFormat{ - Meta: metaformat, - Stats: formattedStats, - KStats: formattedKStats, - TotalSize: totalSize, + Meta: metaformat, + Stats: formattedStats, + StatsKV: formattedStatsKV, + TotalSize: totalSize, + TotalSizeKV: totalSizeKV, } out, err := formatter.Format(in, c.detailed) @@ -202,11 +204,12 @@ func (r *countingReader) Read(p []byte) (n int, err error) { // enhance utilizes ReadSnapshot to populate the struct with // all of the snapshot's itemized data -func enhance(file io.Reader, detailed bool, depth int, filter string) (map[structs.MessageType]typeStats, map[string]typeStats, int, error) { +func enhance(file io.Reader, detailed bool, depth int, filter string) (map[structs.MessageType]typeStats, map[string]typeStats, int, int, error) { stats := make(map[structs.MessageType]typeStats) - kstats := make(map[string]typeStats) + statsKV := make(map[string]typeStats) cr := &countingReader{wrappedReader: file} totalSize := 0 + totalSizeKV := 0 handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error { name := structs.MessageType.String(msg) s := stats[msg] @@ -247,14 +250,15 @@ func enhance(file io.Reader, detailed bool, depth int, filter string) (map[struc actualDepth = len(split) } prefix := strings.Join(split[0:actualDepth], "/") - kvs := kstats[prefix] + kvs := statsKV[prefix] if kvs.Name == "" { kvs.Name = prefix } kvs.Sum += size kvs.Count++ - kstats[prefix] = kvs + totalSizeKV += size + statsKV[prefix] = kvs } } } @@ -264,9 +268,9 @@ func enhance(file io.Reader, detailed bool, depth int, filter string) (map[struc return nil } if err := fsm.ReadSnapshot(cr, handler); err != nil { - return nil, nil, 0, err + return nil, nil, 0, 0, err } - return stats, kstats, totalSize, nil + return stats, statsKV, totalSize, totalSizeKV, nil } From 5f5b49955ceecfc37ba6206c811b998be303f8af Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 10:40:02 -0600 Subject: [PATCH 08/27] Update snapshot inspect formatter test --- command/snapshot/inspect/formatter_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/command/snapshot/inspect/formatter_test.go b/command/snapshot/inspect/formatter_test.go index d74181ebbc..ec02a21a25 100644 --- a/command/snapshot/inspect/formatter_test.go +++ b/command/snapshot/inspect/formatter_test.go @@ -13,6 +13,11 @@ func TestFormat(t *testing.T) { Sum: 1, Count: 2, }} + mkv := []typeStats{{ + Name: "msgKV", + Sum: 1, + Count: 2, + }} info := OutputFormat{ Meta: &MetadataInfo{ ID: "one", @@ -21,9 +26,12 @@ func TestFormat(t *testing.T) { Term: 4, Version: 1, }, - Stats: m, - TotalSize: 1, + Stats: m, + StatsKV: mkv, + TotalSize: 1, + TotalSizeKV: 1, } + detailed := false formatters := map[string]Formatter{ "pretty": newPrettyFormatter(), @@ -33,7 +41,7 @@ func TestFormat(t *testing.T) { for fmtName, formatter := range formatters { t.Run(fmtName, func(t *testing.T) { - actual, err := formatter.Format(&info) + actual, err := formatter.Format(&info, detailed) require.NoError(t, err) gName := fmt.Sprintf("%s", fmtName) From be2a212d75b630641c66b41a2493f39e38cb2d05 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 11:01:44 -0600 Subject: [PATCH 09/27] Add tests for new snapshot inspect flags --- .../snapshot/inspect/snapshot_inspect_test.go | 54 ++++++++++++++++++ .../TestSnapshotInspectDetailedCommand.golden | 27 +++++++++ ...SnapshotInspectDetailedDepthCommand.golden | 44 ++++++++++++++ ...otInspectDetailedDepthFilterCommand.golden | 26 +++++++++ .../inspect/testdata/backup-with-kv.snap | Bin 0 -> 12657 bytes 5 files changed, 151 insertions(+) create mode 100644 command/snapshot/inspect/testdata/TestSnapshotInspectDetailedCommand.golden create mode 100644 command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden create mode 100644 command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden create mode 100644 command/snapshot/inspect/testdata/backup-with-kv.snap diff --git a/command/snapshot/inspect/snapshot_inspect_test.go b/command/snapshot/inspect/snapshot_inspect_test.go index e04494c109..5e5c745533 100644 --- a/command/snapshot/inspect/snapshot_inspect_test.go +++ b/command/snapshot/inspect/snapshot_inspect_test.go @@ -95,3 +95,57 @@ func TestSnapshotInspectCommand(t *testing.T) { want := golden(t, t.Name(), ui.OutputWriter.String()) require.Equal(t, want, ui.OutputWriter.String()) } + +func TestSnapshotInspectDetailedCommand(t *testing.T) { + + filepath := "./testdata/backup-with-kv.snap" + + // Inspect the snapshot + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-detailed", filepath} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + want := golden(t, t.Name(), ui.OutputWriter.String()) + require.Equal(t, want, ui.OutputWriter.String()) +} + +func TestSnapshotInspectDetailedDepthCommand(t *testing.T) { + + filepath := "./testdata/backup-with-kv.snap" + + // Inspect the snapshot + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-detailed", "-depth", "3", filepath} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + want := golden(t, t.Name(), ui.OutputWriter.String()) + require.Equal(t, want, ui.OutputWriter.String()) +} + +func TestSnapshotInspectDetailedDepthFilterCommand(t *testing.T) { + + filepath := "./testdata/backup-with-kv.snap" + + // Inspect the snapshot + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-detailed", "-depth", "3", "-filter", "vault/logical", filepath} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + want := golden(t, t.Name(), ui.OutputWriter.String()) + require.Equal(t, want, ui.OutputWriter.String()) +} diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedCommand.golden new file mode 100644 index 0000000000..b16f6efd39 --- /dev/null +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedCommand.golden @@ -0,0 +1,27 @@ + ID 2-12426-1604593650375 + Size 17228 + Index 12426 + Term 2 + Version 1 + + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB + + Key Name Count Size + ---- ---- ---- + vault/core 16 5.9KB + vault/sys 7 4.4KB + vault/logical 4 2KB + ---- ---- ---- + Total 12.3KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden new file mode 100644 index 0000000000..61e9108535 --- /dev/null +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden @@ -0,0 +1,44 @@ + ID 2-12426-1604593650375 + Size 17228 + Index 12426 + Term 2 + Version 1 + + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Session 1 199B + Autopilot 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB + + Key Name Count Size + ---- ---- ---- + vault/sys/policy 3 3.3KB + vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB + vault/core/leader 1 1.6KB + vault/sys/token 3 1KB + vault/core/mounts 1 675B + vault/core/wrapping 1 633B + vault/core/local-mounts 1 450B + vault/core/auth 1 423B + vault/core/cluster 2 388B + vault/core/keyring 1 320B + vault/core/master 1 237B + vault/core/seal-config 1 211B + vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B + vault/core/hsm 1 189B + vault/core/local-audit 1 185B + vault/core/local-auth 1 183B + vault/core/audit 1 179B + vault/core/lock 1 170B + vault/core/shamir-kek 1 159B + vault/sys/counters 1 155B + ---- ---- ---- + Total 12.3KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden new file mode 100644 index 0000000000..a72fb99b52 --- /dev/null +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden @@ -0,0 +1,26 @@ + ID 2-12426-1604593650375 + Size 17228 + Index 12426 + Term 2 + Version 1 + + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Session 1 199B + Autopilot 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB + + Key Name Count Size + ---- ---- ---- + vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB + vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B + ---- ---- ---- + Total 2KB diff --git a/command/snapshot/inspect/testdata/backup-with-kv.snap b/command/snapshot/inspect/testdata/backup-with-kv.snap new file mode 100644 index 0000000000000000000000000000000000000000..453b3a03f5bd1a55a764ed41b76e770182f66ac9 GIT binary patch literal 12657 zcmV-%F^ZABK^dqCNOglMA!VS3AV{S_m7uOTeQyI39frP%C0ftL3909z<9p17 zg4Xv{@q)@g)hJ{~q7#D(&{t082Q>z%o-4n!_JbNT9382E69`p;{{Vr2qvb&q8iWz> z2n@^-K;vKvAc6qH;2h=AcqD*Fq7sNq_@65@IzR_yam4>Q{i6{W1o~I~BmUO^e@6+* z*8^zu6Hr$$0!7_*cqTX6Y&(PyDSMH=PjG`K{^n09Zw$ z2dIAp_$TOP8e{-R0!;uX15c1jB@)2e3?22H)w5;k63K+LTY3~cKqC1Vflfpk9i)N; z721y`YqOTYg%(i(N4mY|>gbb>s-%HbM|*cFg-*d!NcMOJl?syS>BzF5?QTSJI%2&I z@J`3%{%9#1A*&#Z$tP3D_MQ|vNX>QxNHlP5%-FLGDO7rk7%AJ3O7Zpy%UK;Esw>n% z8l6a987F?Yt)rfH{a5QTzzs~x(eQSs(7@myxvkAM1nADR>hcx8e^mSwg4mro8U z)CXzKS^ypN0(^q9^eF@|E#K6gMyG;+8?E|}{QQXMdNNmMnE+0->RfY>=;TbN1?L(9 zG#ZiY#FT3eP|3?*{+7ePC5LIKWan>GlL=+5gD0y?hvNg=<6Y|d995dqq=GigOD zUw?!>ElZU`@JYL;4pM(mN{vdS6W8R`kq%NJW$F~XD@awNkm&%C3{rK})3UUT4As&K zmj9a4KodRV+AM7o6T`F|Z4e-U)apE)WpR1}BuI`vNOz_X(uxfjbaw__AMjqDO+yNd z@l>KaU4`mItIjrd21%r}OcOQ3v|}&i5?`^gy;rR7|Uc@+cPh- z^O`Y8brf3-fS55IpfhNHA?xg86go&t%UOkRQ^g4+)5QQ1(GwH{$OJJe2;iNUrI`N@ zH59Iuz#sptbfr0~r^$6+WlWZdk2{$5zqOnC-#yv+lg6(gl7H3hDXkb`G!P(x1Q>p~ z?$3Q_%>Va!-XdJF!t`>pkBtBwZ&LCNA^(c`h zMkI>?WHAPLHO6AJm3CKDfd|N9I1o>9TNa7fYTqgLk5)nI4>^^r+Dv~m@ed9C=hvutnzia@?RhI%E%@DwU22QUak`U<|4itBKtC@P_dZ<&{WnGT-QlwMS?wNC8I+9lz9?0~5-?`HAcHz`t~Bj??%*_^8aV&}@G7$w8ZQgeN%_+FV$2Qn_lP3?1d zV41Em=%}2}S-k&12(^=D1w%x0%cP zg=-osne#boC<(GUZrFGiducHi@+BTu`{5Dajlx?eJh?ra!SRd~+`!|MbIeR5+B087 zJkz*Nn?Fu{!U6rTICQ*~ddFuy-g9g4-u(--q~(Qz9Fgou`7P9Qzd$9gK%M=9Eq|eC z4>zsxX;4Xo@jwfMEr~DgR_;h1clo~aqJ#cWz{VMkR!!i@2bj{H3nz*SGfFx&ne1*S zRM`6HtLbZ-O&;r#xWBJN{S+ol;D&|F2(rNzZBQGM9)C>6fUs+o+_2o=yRkH7l*BS2oV{& zvv_ZjG-N&as5RiPt%2?e`cR2vr{6-2`UUFHN)|qb@cODMIVGCc$`vU2*&w@NZN1eV zt|`GU(iV;}7e(vGhc-LpstMVoB(rHsq&_^9FwBDI`W}6rkvSbZ( z*|u`m&ct=xEo{-S&(R5$`6$94;Vt#$ar~|4xCDpmWh{PCh4_M2lI~F1#-7~UExanY;q7U+(SC0vq!c{>yRPzi!yzXvv!j0!AFfo8`s@-8c#ZRE%n$OIv4>x#BnnT z&X_kE7iAF;-A=muDmz6Fp>fw@w&zier1Ad#Md}d-Ry5USTih{}&VcR4q7BY>l|V_^ zUc9?dH^;*Xy`w{oJG4ZDUk6BCJkSTrQnX%h)NEENJ8}+CYdxLwxi9iSY5k-OdE*sD zc*ldHOG*{Hw4>TDoK{pVQ&W+xJ9m)vJyXz;9HOcVFTx@?H~yidtI;O8$6S>k^gMRC zFHkLLeDdX9j~3osbJbX8ioNKK68blbvK*9c*SeWRU0 z8resAs!bwuTi*=l-EckZtuCS8#-1C78q(YK0ZCwyMc6hvety?K%rH(di1{G|4 zLsM(RcLVWEmaxFdGt9R22aHN^Uh+CF)IEKWxDNSbeOdDQpHl}9ydv+6cdNba3TZxItA@=KRu1m$+>h24|o ze8S%EKDKEya{ZgG?%$mvSkvUw8J4d9gwbMuv#5Nfi-fdU7;#*8AwJ zn9{uh^-t-2hIQ!a^j%qD=`RYz1a35cBJnh)^GaN(+B#-_|NPye-h&QWRfc>}i^9iU zM*5L58F*bL`ErF8%=47#oV{_%&yLt~q@LMS*M=jrjQ8BWF0wT;Zq8z0(wX;_Pj5Vy z)28=pXdf}vi#J8mF1^-ZE>~0JL{{DBfXkd#_fPMWe|Q_pD{HoAM)~MD?eJZP8O~7e zBa4PiAqEPZq;2trIoFIa4_h6sPuY2dWZZKq=|Y%gd{@ldS{s2c;!7Oun$+B4$r53A zYp4gAqoDgPT0ZJBsw5%FS0&-EOJ5+AS>a5b^rJa~EY+!V#WJCi=p12jOxVedMn{e$H@PQB}5es94RiB^lL@7qb! zdBlxK)kr=ppYTPy8}W$!f*k9W_tz6)ru8>)icej2rqvto52nxiC0fev(wOb#$a=Vs zt<5gxh>9pra%7DT&wJf#m)>bvDL3mp7tyQk9^rjm_w=Rjg^MR3k*zsrhMQgbl^0!1j5g!gx|93Y8!m`r1)rc>)Cc=L>_>-z*Fm zQ#dS_w6DBO6N`l7H}>a~0%xoNQa}Fc%Mihr>T!74xM@RNn2fLYK~E^zk7GY1>+T(k zi!dPr#XP@yOI*uVM60@rS#`f=#SUFwhGcZed&X$}df->q0I&QD;QwTKaOD@=mMgdi zYd*d-MI0BG5~AH3kQQhbvdMD3qIOEm(pu8F^r9{>v9}>lzv`S;Bs6pSr0pL4eY94V7pNmWcu&tWr@G6 zYj7*T6_c7z@B2Ey{MI(&S|d5H+eVQT#qBD$$MM8FkcJD)hNfpWFjdY-4)^P%;j=s}&~FFH|NqGk@m2uopq6T5K_ePF|`R)L8= zx$!WIFhMs{`gN^NmM>`hkCf>PqPGwicMbGgiTDgoJ-2pgPfvi>Ry{M13MmU z_C#K8v+(JR)>3%;m6NADJ1(Ml>jleC#}BaXc>NkP%DXr}>rcIbE6`8G26q#HmHj!`DB2kXrBT z#osym&m~o~-dSdy6=&sqr7K;Y%)M*76-<&HOK(3Sp?>((6XVZH|OHq9yjW@o!^jSnc za@q@xB3l>J&Iy0-c=kyq{k$>v)_}0%lWeow)lmVN3wO0jKX*vwAKj?Mp=^Dakfhf- z86or7*<N1_r*_~g)Y)Yi;vO3# zHYjSz?DB_jZ@DVc$lPtYH7#)PfN=PZTB?d3k7c?^q(-dOgaE^8sr59 z79v?82NrsGa29#|{LaKJunDRo`cY9@_|c$1r$fea?MkCHO*&rg^;tW!e3-2s4sx{f z+Me&;Cpw@XJgqRobHmfAb+i3vX0*EtVV|$v{>!ubXZUT07IcoCTrj-yA>!E^fnBxC zHw8j89kv{jd%_6qa`urWoy~2SQw{na+_Jqi_L4}GRmO$cu&(=?ARpO1gCtb9HgOJ& z94I+|;%U}?fx4!JDW(+q(Wgw^QAfG14?JL5kRo9^7
CX7-le$TjKZssV6LF+Z#|0p z#Uqs!F z-5u^$|5n_qcwX4!s@^&H2fnh=zQD_QoU*3mSD!gFq|$eMXXT38@MZJ66ob-FyTw_O zcpUFA%mfS3le$uq-Gd3Zaq*`q6AQB5mp$zygC)uk%8G5?Uax0t$eNBN_>emfjmmJ1 zTwh*17N>7?V!Fl@-4bhmovPn<{PV|dui#AyZN#nTuT>O%7d3x%@^$umrOG(Vcxk!L z@wAr?@)z^ii_iOA{48JIvV%#-Z20+$%iYn^^qOiOgbK z#<7qu-1j%d>yWp9X+eyQsmIn}wpF#;*iEq-g?lfgBf%&7fp)rDZ=b|z;&%^3^7A}Q z1om5Qs?&H~D_(m*udd+2$O)F&1ryuz6eNMnX=|Eduex|PpZ7!aC3zqD>KkWIJaU_O zLcNw*Xp|Pe#3XQR+oRL9o26dHt0NDEx+TLmoRbtCW!{f+`qqo$)k9=V8KE!0#;|5advAf;ZqH$=&iY^>f`Rj{Ckx%xmGvJ z_9Ym5ot8kHJij$(!gV`!J@HC@x*X%YN<9+AiHHYCa&ScjMNnQ5gu$_R0t|zaN5k+6 zcsLA=#}cse7#tpfSCm_>0U$smOrD^CgCWogNEl8YAixwrMR^2H0f$x~ILZN3yfe`g zlyd{=00E!_za?_XFEhu16(W0_GynsxHu;XritQ(3r7Cr>$j)QehwS#i%e89)&7v21 zy~FbpdH2SPf1bMZaNoO=&VFBki|VF!&YcetLB$M18?(t`2MxtN$od_7=3~xry4=ls zV%i)qcClkY&T=r6rAO-J{=~B{e0zX52#vH;{ZR6&6vg&&1ytST{FBg4@;-igot3pc z=O4;6h_Q?IPv2otn%Gknf|q-A_S!erp(b&Nv-p(QOPd-WNiSDHL=&gTv#)Ft16ioA zNn3PV(rWbb?LRf2WcPEa52ELWWM&(^tK4s6Bb#SjYjLR~o_P4vi5lqT_^+pfBlf-@ z+|N17^5rHS8nNl}*#%mV#5b=?$T8E%JnhFdau8eliX+Fi^FNk+?{OS$9pD;HGmF#Y zRq1-|Urx)_JP{bs)}$*<{2WzQK7LlLNzxkft<&>!ZBLJ8+Xu0|Es+DjLlw2mXE_Du zp-<0g?-QAR)t<$mKYe&9yY#cPSqL89h#5a~h5v&=Ky%#e8D2-=bGBsrWA$j7tHoDl z`Jnnof)DwgOy2Gkzqu8b|IyL+`%sL@ps9J&SEay#}#QWgHi^EZK^BL3ky_+u-#B#Jt<3nBBj%Rij?jY5)tq7 zUvsi_JttY1Ryg@6!Pfj{@5nR7s3+a*GuN5~FL<>Jjdt$76|20cvSG9I8;Y>H2hrgE zok$B=uOkYy2Cg*di#}V<$G5iac5f-y`^42dt-Lq4He>$0>iyT;SC}E2{Uo%D$9fzi`tDCe>>QhSwcI7? z9FwbE$K3{Lzrht8nYy*-rHuJ*qvlY@6H1+(>GI(-hwI)I@qUb#QY6pTCUZj0PRX!u zBbt0HW>)Pmk-Zz387>$#VrgQco>a_{&OC8tDI5{i%n|(RVo>Vk_DhFNjSStryeizX z>W^@gd*2Y zzsF>k-3fd5?LJw+8PZ)4MJ>?p-)Ox>XlB!Z{oFhCR{iF=T|LPus59C+*ZGX!#?MaO za=o-4>+D~+d#djnCFS5j!}>86$BrR!_bvMMLZ2(VGi>a(63w#s)bJ*L1it`*`p_lk z7u2*Ec9sir{u6%whwXXK#9OC3Dq*F9K1gY5Q{O~T0_P9E`z)jBsj-O3HHxE~Jqqxv zO&{72!ruCfPfE#NYr#QymmT%%XmtfTB@H+p`$b_YIc8XSV{mN4V>bUGB)7*+74z?A z2Zz4A54tr%NFO|jy6hm1Fuk4UXn**#-`$q2MO%(yrmEfF-nd9+L_|7DO&`^h=gpPj?t;RppBRsn{>$fICr z1OWwu<54IW9F4<+SU8@5#lhw901c#pWEzo9^d!=MKaVT;&27cER`@`hnp;kL_S{#% zKQW|VwhCmbP;MX$eN~?2yRhvBzGaKNr6now{-|xE_POlae4jEo)%W$eZ8166;xF9s z%U!H%KY6rg)gzh@O%A`j3I(Y&IV$ME0BLlZ91@O%%OMcIgA5P-1u|>}GF{92tYeJg zv^r0wYiYJ7@0O4Yq6DEr*dR7fRI#0-M8E>A(m#|pz8>18HPG(;1K5ww8c!k9sT2~- ziArI(|2B-5e!(=qg2^f~GG_5LF-==kO2*p2b(i!Ech3$!lOq23hi^weE;aTvd^%Tf z%z1u_#jW_rv+1+#`*_-W>bvdds6%H;39XZ<#P^LNnfAv_E6aN0zFNtLWSv^Fb%+t_ z^Z152GyX{V;T6gImv19E_I%N8@XdD$u`XooNgPb*fGzpZ<}$u)pviS*YK9h$SLZ?4pGgJ15M{HyQ#lyNhv)} z3GVM;QEV1TV2aZHINBA%eNQM=%}ISH>uw#GcslB9 z-O`>O(;R%pMZ-yh*dtlv{CB$z^9(*x$h78~9*?jOrk4>s7v@`vj!apqa$h=b==qM4 z-Gi31UCg@-Es9!{k7>@HWt`WU;p8YT*OWOe&8KP8(rZ5S&0^-Utxn0CW_gb0mCs&? zADDy=$?e+Wgu4ed60%}GsJxFfDwrwgYd-C|*STGI{;7df3SS7nurDgaTfvRjllKL) z=1u`koZgP5`Peg2r#`;hnG_xEEUJ;=#24>eyamugKB{QV4b2Y?ay0==%cFL>d*QC^ zdM=V2068IWiev8y(>{)%i?^?cvF#_O_F3Z|7h+2-VqMkA3$_VmTcQq*R4^OQTroSe zBo>y_y^|teb%V#?<}0nw&6mGBmR5(}LmbqE^gf;yJ<~WlxT&-w>rRqzpM8B7BHI2Y z=wv4Cx50+Bb>OX+)>#%m{fLMQ*0*1H>TQeKyKv2=)=J0>9SA09Z)$if?K{rYpbWo$ zuU7S;9|cHsr^oEbv9}ILf@N6`~-7Ck#v$e-FDvL#TARCcy6Tr znIx+_tSs96N-fO%3r|4vw-HImS0WB=-yVGo#g0L9G=xawzxK{_T zIv=d4O({_x=F9UR>9&c@ywo}!O7~U4iBw%-zQXr{U13AOj*fhl7;mOKMwz?qVhA&Odl&ld)h!ZuH4F3z!{E;r0dojz*s%3C%e;}|AbB;C7T`{jnr zL-adW&JJFQqz<>ZRz$O$cUAv3-a0=#Z-+Ws6?*4O^`}WPFGj2JOZqr4 zyuRiboOsI$ApOSd_{;n9v~17x97?=<&vUlAtLZ8HY**kxFS+`tJJf?a?{Sn#+G%Ya zW9_wnBkR3ag2QBgTk)ozk3$MeI&U^}skd+D`7Spm#vkT{3x~df1w44c;c+LY$vd(Z zp5J|K_w=ohst_$xN{JcRA~mNO)u<3U{-}FfrT1r=h5Kzj}4r1y<2N<8E+?6 zJm4|WS@ak7qu)4sZ89omn-$h?(f32;vu>U~tA(K32?b%C@oFfZEjF^SLOhyUr>tY( zcH;A;9T)aAFAlRGU>s%JHs_fdq-PE9J#hQsID0`T&yKu8iQPLpSWM@5<8QaWvYjHE zZHU(0!w=siiBR4k%|>su{$lk)6j9dp#&(ADYLM=Z{WF$=rM09jm6_rvzI%DyN=<$H zMrVuq>AhQ18xFO`k1X08v#Wfp71+hX%@V5KeUvVIO58k~)yDhkGwH^gk4^afYFgDT z+n?<$g@$?5+7!N3GvkyF?dWW5H`SkZ4 z{gz`#ThfXc3zgsVW?z>b5PSUD+QKMC)dhdreM%0+bMnH6h*QOqOV;1)CB>|x4r8CtG^<|Vmi01NV5d{<2L z#g}KdKff)z|2xAvPrz24kRUSiZFc*HlC%-F&G1V#hG(i!vnmH>f)?|i-qY+8Oc_pl zJdtoJ;KT5X+(zfTjc2z@dRgI#`65*<^ESs1s<B_9p+`n;Pa=a=VuEdEpH2*yj|3y z^Fl6bucALbY@FZVEEeqp+1{JKx(7S>}bX?G+KviX?WWuc0%>1hQrxK8D@ z9KYs*KFQ+VHe_0PV4onSbp?80PT0K)#OYdhdIfMGp(IDo%E%ABg+bES<< zF@5uJ0m^E+-l6qv(Su0;cZp$}kNZV$B5m$}nU4uHF<`T_G5>Z&KwHHC6(-%%d5!OQ z=f%th!$>hMqVO{jH|zXCqy+2Zssg^cu5RY^0zQo9u-+6lbwl9JXrUH42iM|KrzPWE zZH|lW-aZ}UkS|9#6mQUzKe%oEg3wPLbU)nMgOyEr$)^v=^PZ@?#s5kgE0g_JHN+O5 zsEG9X$nO>C4G?#hPTyoYh>Kl6vrPW0=la&nEvX>QokFI8us^P9|8|y{{LB1(cx9F; ze2_ivc3P)g|LNOS530{qTk98?UhJX!?kH@PV-`cSsx#eSc@!85ygTBtS4od3Ad*Yc{XTwT(NX z3VOwk$G#XuQ{pOJf?LXZsCpxO9|JWvrba;A&tS6e_ijSrZBtOCOEw=vK7CFbEPF`t}zaYIV;p_lmmlVl%i zyPL~!xWla{uFr6qfDt1ldotlh8|B`vspl7!4>X**pVce-ozS{ixGU87iPv5x=F_2V zM)5(c(W3pEWobKWn<5HzFQ~_)9>M_&`=zVfXL7t4`{m3VB0@J1jH8-+KdD@bPTQ)% zyx8gy3-jnYZpale)Axl=SRa+lpQ2*2J}N|loHGWl;7E{1JIW&w2s{Q0AaN)Z7K=b) z;0p2p91G$ckq9Ibi^rgFNW21G9tX$B%Yz8GA{GE}1f0Te1C{y9CF-Wt$IP$npVv~B ziO&k?=03kboCr6)`AMKy`~KwU`!CSf<0IH2!DpV3ZjNNcVIr{+d*I0~QyCdfO~FO* z>`PTz=w7Lfw+95h^?>Z-Ib(3+2afk%8MeMSe(=1-l#w=UH_M#=mE4*Qy8TM;ltWn! zxj}gc+}M^oY{n<1eFrt4_S-7opM@B|sRwjkBHlE8#NB>bqg#}iJSW=S$aNJli;7Wb z{5nFuS=9FJpc0!i^Yf$~7wc=ztn{0`5wuU?B>cknE#a(pOFQp|H z=MB9U3gBIz1hWLqOHr2IlLpg zf%-(l&#N0jwewL;3OE&(IWsc1vv0W2LvADcO#>CWieDm!M}E9jQd|SUG*#>1;aZwR z{mxcXemhl7C>HX_b-M4yp1lt?vUrA`%T}z6CP<@ZKTYwp42_=s;>1|bwDdJht60x78t_k+ z%IUwkmV-cJ+*~k1a{J#P`}W{HW@GtZ%|5 z=6;fSN^!NLf8S0vjp6?L=Puyk>s#6o zeoc)80%V4}ItUPcY%)@}wsqDjbs`PGkwDXx3wWSD(P?#aklG48fJR?=RSZuz`Qz^I z6Yh~(NU1*HZ9)ZzBqG^KkK#mYUdY!cl8-YeREFD6h*-;vKmvnI0LXYBD+(Dj0i4pR zR#6z!0UVJ;^j$fZ3DFIF3V}eFO}GakWj~Po*otNOH=?K3y!MoqVc?#_aHoM(dbTkg zpo5`n{b5`8f>hC(*K=t?XVVg4R#RivKvaqJ(w8Dt;WacO{YuvUDGJA|4>E ztCRPm&L22<^Z&H3Q0h;6_Nu*rPI1H0=oIold=1|jl&9v*AiL@V?t%a3cjx%e7h`P| zBnE42s&D+CJI8;u{}U1gU$_4Y9D|1cz5mO9N0GHv8fJnSN5=5XBacB%2MGzDFb2p;@ fAMeQYclo>gUH&frJ>~xi00960IqUaW08#(|Oz9n# literal 0 HcmV?d00001 From 1a50aa023a9ef554ac27b7931e82d3ba4999c5ee Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 11:02:02 -0600 Subject: [PATCH 10/27] Fallback to alphabetic sorting if size is equal --- command/snapshot/inspect/snapshot_inspect.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 1c1938dbbc..6faee4d38f 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -169,7 +169,14 @@ func generatetypeStats(info map[structs.MessageType]typeStats, kvInfo map[string } // Sort the stat slice - sort.Slice(ss, func(i, j int) bool { return ss[i].Sum > ss[j].Sum }) + sort.Slice(ss, func(i, j int) bool { + if ss[i].Sum == ss[j].Sum { + // sort alphabetically if size is equal + return ss[i].Name < ss[j].Name + } + + return ss[i].Sum > ss[j].Sum + }) if detailed { ks := make([]typeStats, 0, len(kvInfo)) @@ -179,7 +186,14 @@ func generatetypeStats(info map[structs.MessageType]typeStats, kvInfo map[string } // Sort the kv stat slice - sort.Slice(ks, func(i, j int) bool { return ks[i].Sum > ks[j].Sum }) + sort.Slice(ks, func(i, j int) bool { + if ks[i].Sum == ks[j].Sum { + // sort alphabetically if size is equal + return ks[i].Name < ks[j].Name + } + + return ks[i].Sum > ks[j].Sum + }) return ss, ks } From 3ee20d500af4d1e064da7e10dec258bd44fb8948 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 14:26:47 -0600 Subject: [PATCH 11/27] Refactor to reduce how many vars are being passed around --- command/snapshot/inspect/formatter.go | 8 +-- command/snapshot/inspect/snapshot_inspect.go | 70 ++++++++++++-------- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index 0531623f5d..bd45501ab4 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -15,7 +15,7 @@ const ( ) type Formatter interface { - Format(*OutputFormat, bool) (string, error) + Format(*OutputFormat) (string, error) } func GetSupportedFormats() []string { @@ -38,7 +38,7 @@ func NewFormatter(format string) (Formatter, error) { } } -func (_ *prettyFormatter) Format(info *OutputFormat, detailed bool) (string, error) { +func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) { var b bytes.Buffer tw := tabwriter.NewWriter(&b, 8, 8, 6, ' ', 0) @@ -57,7 +57,7 @@ func (_ *prettyFormatter) Format(info *OutputFormat, detailed bool) (string, err fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) - if detailed { + if len(info.StatsKV) > 0 { fmt.Fprintf(tw, "\n") fmt.Fprintln(tw, "\n Key Name\tCount\tSize\t") fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") @@ -82,7 +82,7 @@ func newJSONFormatter() Formatter { return &jsonFormatter{} } -func (_ *jsonFormatter) Format(info *OutputFormat, detailed bool) (string, error) { +func (_ *jsonFormatter) Format(info *OutputFormat) (string, error) { b, err := json.MarshalIndent(info, "", " ") if err != nil { return "", fmt.Errorf("Failed to marshal original snapshot stats: %v", err) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 6faee4d38f..c40fe3d762 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -63,6 +63,16 @@ type MetadataInfo struct { Version raft.SnapshotVersion } +// SnapshotInfo is used for passing snapshot stat +// information between functions +type SnapshotInfo struct { + Meta MetadataInfo + Stats map[structs.MessageType]typeStats + StatsKV map[string]typeStats + TotalSize int + TotalSizeKV int +} + // OutputFormat is used for passing information // through the formatter type OutputFormat struct { @@ -114,7 +124,7 @@ func (c *cmd) Run(args []string) int { } }() - stats, statsKV, totalSize, totalSizeKV, err := enhance(readFile, c.detailed, c.depth, c.filter) + info, err := c.enhance(readFile) if err != nil { c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err)) return 1 @@ -135,17 +145,17 @@ func (c *cmd) Run(args []string) int { } //Restructures stats given above to be human readable - formattedStats, formattedStatsKV := generatetypeStats(stats, statsKV, c.detailed) + formattedStats, formattedStatsKV := generatetypeStats(info) in := &OutputFormat{ Meta: metaformat, Stats: formattedStats, StatsKV: formattedStatsKV, - TotalSize: totalSize, - TotalSizeKV: totalSizeKV, + TotalSize: info.TotalSize, + TotalSizeKV: info.TotalSizeKV, } - out, err := formatter.Format(in, c.detailed) + out, err := formatter.Format(in) if err != nil { c.UI.Error(err.Error()) return 1 @@ -161,10 +171,10 @@ type typeStats struct { Count int } -func generatetypeStats(info map[structs.MessageType]typeStats, kvInfo map[string]typeStats, detailed bool) ([]typeStats, []typeStats) { - ss := make([]typeStats, 0, len(info)) +func generatetypeStats(info SnapshotInfo) ([]typeStats, []typeStats) { + ss := make([]typeStats, 0, len(info.Stats)) - for _, s := range info { + for _, s := range info.Stats { ss = append(ss, s) } @@ -178,10 +188,10 @@ func generatetypeStats(info map[structs.MessageType]typeStats, kvInfo map[string return ss[i].Sum > ss[j].Sum }) - if detailed { - ks := make([]typeStats, 0, len(kvInfo)) + if len(info.StatsKV) > 0 { + ks := make([]typeStats, 0, len(info.StatsKV)) - for _, s := range kvInfo { + for _, s := range info.StatsKV { ks = append(ks, s) } @@ -218,15 +228,17 @@ func (r *countingReader) Read(p []byte) (n int, err error) { // enhance utilizes ReadSnapshot to populate the struct with // all of the snapshot's itemized data -func enhance(file io.Reader, detailed bool, depth int, filter string) (map[structs.MessageType]typeStats, map[string]typeStats, int, int, error) { - stats := make(map[structs.MessageType]typeStats) - statsKV := make(map[string]typeStats) +func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { + info := SnapshotInfo{ + Stats: make(map[structs.MessageType]typeStats), + StatsKV: make(map[string]typeStats), + TotalSize: 0, + TotalSizeKV: 0, + } cr := &countingReader{wrappedReader: file} - totalSize := 0 - totalSizeKV := 0 handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error { name := structs.MessageType.String(msg) - s := stats[msg] + s := info.Stats[msg] if s.Name == "" { s.Name = name } @@ -237,13 +249,13 @@ func enhance(file io.Reader, detailed bool, depth int, filter string) (map[struc return fmt.Errorf("failed to decode msg type %v, error %v", name, err) } - size := cr.read - totalSize + size := cr.read - info.TotalSize s.Sum += size s.Count++ - totalSize = cr.read - stats[msg] = s + info.TotalSize = cr.read + info.Stats[msg] = s - if detailed { + if c.detailed { if s.Name == "KVS" { switch val := val.(type) { case map[string]interface{}: @@ -251,7 +263,7 @@ func enhance(file io.Reader, detailed bool, depth int, filter string) (map[struc if k == "Key" { // check for whether a filter is specified. if it is, skip // any keys that don't match. - if len(filter) > 0 && !strings.HasPrefix(v.(string), filter) { + if len(c.filter) > 0 && !strings.HasPrefix(v.(string), c.filter) { break } @@ -259,20 +271,20 @@ func enhance(file io.Reader, detailed bool, depth int, filter string) (map[struc // handle the situation where the key is shorter than // the specified depth. - actualDepth := depth - if depth > len(split) { + actualDepth := c.depth + if c.depth > len(split) { actualDepth = len(split) } prefix := strings.Join(split[0:actualDepth], "/") - kvs := statsKV[prefix] + kvs := info.StatsKV[prefix] if kvs.Name == "" { kvs.Name = prefix } kvs.Sum += size kvs.Count++ - totalSizeKV += size - statsKV[prefix] = kvs + info.TotalSizeKV += size + info.StatsKV[prefix] = kvs } } } @@ -282,9 +294,9 @@ func enhance(file io.Reader, detailed bool, depth int, filter string) (map[struc return nil } if err := fsm.ReadSnapshot(cr, handler); err != nil { - return nil, nil, 0, 0, err + return info, err } - return stats, statsKV, totalSize, totalSizeKV, nil + return info, nil } From aabb537aed66c1e456871e5f0727632bbd5b9f17 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Thu, 5 Nov 2020 16:25:21 -0600 Subject: [PATCH 12/27] Pull sorting into separate function --- command/snapshot/inspect/snapshot_inspect.go | 38 ++++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index c40fe3d762..ee1c87dd05 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -145,7 +145,7 @@ func (c *cmd) Run(args []string) int { } //Restructures stats given above to be human readable - formattedStats, formattedStatsKV := generatetypeStats(info) + formattedStats, formattedStatsKV := generateStats(info) in := &OutputFormat{ Meta: metaformat, @@ -171,22 +171,14 @@ type typeStats struct { Count int } -func generatetypeStats(info SnapshotInfo) ([]typeStats, []typeStats) { +func generateStats(info SnapshotInfo) ([]typeStats, []typeStats) { ss := make([]typeStats, 0, len(info.Stats)) for _, s := range info.Stats { ss = append(ss, s) } - // Sort the stat slice - sort.Slice(ss, func(i, j int) bool { - if ss[i].Sum == ss[j].Sum { - // sort alphabetically if size is equal - return ss[i].Name < ss[j].Name - } - - return ss[i].Sum > ss[j].Sum - }) + ss = sortTypeStats(ss) if len(info.StatsKV) > 0 { ks := make([]typeStats, 0, len(info.StatsKV)) @@ -195,15 +187,7 @@ func generatetypeStats(info SnapshotInfo) ([]typeStats, []typeStats) { ks = append(ks, s) } - // Sort the kv stat slice - sort.Slice(ks, func(i, j int) bool { - if ks[i].Sum == ks[j].Sum { - // sort alphabetically if size is equal - return ks[i].Name < ks[j].Name - } - - return ks[i].Sum > ks[j].Sum - }) + ks = sortTypeStats(ks) return ss, ks } @@ -211,6 +195,20 @@ func generatetypeStats(info SnapshotInfo) ([]typeStats, []typeStats) { return ss, nil } +// Sort the stat slice +func sortTypeStats(stats []typeStats) []typeStats { + sort.Slice(stats, func(i, j int) bool { + // sort alphabetically if size is equal + if stats[i].Sum == stats[j].Sum { + return stats[i].Name < stats[j].Name + } + + return stats[i].Sum > stats[j].Sum + }) + + return stats +} + // countingReader helps keep track of the bytes we have read // when reading snapshots type countingReader struct { From 5dd703a62a188456240f4adaf548b6bdc3637aea Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 10:09:03 -0600 Subject: [PATCH 13/27] Move KV stat gen to separate func --- command/snapshot/inspect/snapshot_inspect.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index ee1c87dd05..1717e1d8b5 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -145,7 +145,8 @@ func (c *cmd) Run(args []string) int { } //Restructures stats given above to be human readable - formattedStats, formattedStatsKV := generateStats(info) + formattedStats := generateStats(info) + formattedStatsKV := generateKVStats(info) in := &OutputFormat{ Meta: metaformat, @@ -171,7 +172,7 @@ type typeStats struct { Count int } -func generateStats(info SnapshotInfo) ([]typeStats, []typeStats) { +func generateStats(info SnapshotInfo) []typeStats { ss := make([]typeStats, 0, len(info.Stats)) for _, s := range info.Stats { @@ -180,6 +181,10 @@ func generateStats(info SnapshotInfo) ([]typeStats, []typeStats) { ss = sortTypeStats(ss) + return ss +} + +func generateKVStats(info SnapshotInfo) []typeStats { if len(info.StatsKV) > 0 { ks := make([]typeStats, 0, len(info.StatsKV)) @@ -189,10 +194,10 @@ func generateStats(info SnapshotInfo) ([]typeStats, []typeStats) { ks = sortTypeStats(ks) - return ss, ks + return ks } - return ss, nil + return nil } // Sort the stat slice From fbe850325805c30bd1ae57b40ea830ba4ac5a7fd Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 10:29:49 -0600 Subject: [PATCH 14/27] Break KV portion of enchance into separate func --- command/snapshot/inspect/snapshot_inspect.go | 79 +++++++++++--------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 1717e1d8b5..be4241dbaa 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -258,41 +258,7 @@ func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { info.TotalSize = cr.read info.Stats[msg] = s - if c.detailed { - if s.Name == "KVS" { - switch val := val.(type) { - case map[string]interface{}: - for k, v := range val { - if k == "Key" { - // check for whether a filter is specified. if it is, skip - // any keys that don't match. - if len(c.filter) > 0 && !strings.HasPrefix(v.(string), c.filter) { - break - } - - split := strings.Split(v.(string), "/") - - // handle the situation where the key is shorter than - // the specified depth. - actualDepth := c.depth - if c.depth > len(split) { - actualDepth = len(split) - } - prefix := strings.Join(split[0:actualDepth], "/") - kvs := info.StatsKV[prefix] - if kvs.Name == "" { - kvs.Name = prefix - } - - kvs.Sum += size - kvs.Count++ - info.TotalSizeKV += size - info.StatsKV[prefix] = kvs - } - } - } - } - } + c.kvEnhance(s.Name, val, size, &info) return nil } @@ -303,6 +269,49 @@ func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { } +func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) { + if c.detailed { + if keyType != "KVS" { + return + } + + // have to coerce this into a usable type here or this won't work + keyVal := val.(map[string]interface{}) + for k, v := range keyVal { + // we only care about the entry on the key specifically + // related to the key name, so skip all others + if k != "Key" { + continue + } + + // check for whether a filter is specified. if it is, skip + // any keys that don't match. + if len(c.filter) > 0 && !strings.HasPrefix(v.(string), c.filter) { + break + } + + split := strings.Split(v.(string), "/") + + // handle the situation where the key is shorter than + // the specified depth. + actualDepth := c.depth + if c.depth > len(split) { + actualDepth = len(split) + } + prefix := strings.Join(split[0:actualDepth], "/") + kvs := info.StatsKV[prefix] + if kvs.Name == "" { + kvs.Name = prefix + } + + kvs.Sum += size + kvs.Count++ + info.TotalSizeKV += size + info.StatsKV[prefix] = kvs + } + } +} + func (c *cmd) Synopsis() string { return synopsis } From 5ad0db73c8c8d67de98b0606c4f76666280fa2d3 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 10:35:24 -0600 Subject: [PATCH 15/27] Make docs for params clearer --- command/snapshot/inspect/snapshot_inspect.go | 6 +++--- website/pages/commands/snapshot/inspect.mdx | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index be4241dbaa..97c2db3541 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -39,11 +39,11 @@ type cmd struct { func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) c.flags.BoolVar(&c.detailed, "detailed", false, - "Provides detailed information about KV store data.") + "Provides a detailed KV space usage breakdown for any KV data that's been stored.") c.flags.IntVar(&c.depth, "depth", 2, - "The key prefix depth used to breakdown KV store data. Defaults to 2.") + "Must be used with -detailed. The key prefix depth used to breakdown KV store data. Defaults to 2.") c.flags.StringVar(&c.filter, "filter", "", - "Filter KV keys using this prefix filter.") + "Must be used with -detailed. Limits KV key breakdown using this prefix filter.") c.flags.StringVar( &c.format, "format", diff --git a/website/pages/commands/snapshot/inspect.mdx b/website/pages/commands/snapshot/inspect.mdx index 0b039f6af6..c4ae783a4f 100644 --- a/website/pages/commands/snapshot/inspect.mdx +++ b/website/pages/commands/snapshot/inspect.mdx @@ -109,7 +109,7 @@ more details about snapshot internals. #### Command Options -- `-detailed` - Optional, provides a key space usage breakdown for any KV data stored in Consul. -- `-depth` - Optional, used with `-detailed` to adjust the grouping level of keys. Defaults to 2. -- `-filter` - Optional, used with `-detailed` to specify a key prefix that excludes keys that don't match. +- `-detailed` - Optional, provides a space usage breakdown for any KV data stored in Consul. +- `-depth` - Must be used with `-detailed`. Used to adjust the grouping level of keys. Defaults to 2. +- `-filter` - Must be used with `-detailed`. Used to specify a key prefix that excludes keys that don't match. - `-format` - Optional, allows from changing the output to JSON. Parameters accepted are "pretty" and "JSON". From 1ef259b093678586d389937b221a5c5f5b56ea4e Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 10:44:09 -0600 Subject: [PATCH 16/27] Rename params to better reflect their purpose --- command/snapshot/inspect/snapshot_inspect.go | 24 ++++++++++---------- website/pages/commands/snapshot/inspect.mdx | 6 ++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 97c2db3541..3ddd30300f 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -31,19 +31,19 @@ type cmd struct { format string // flags - detailed bool - depth int - filter string + kvDetails bool + kvDepth int + kvFilter string } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) - c.flags.BoolVar(&c.detailed, "detailed", false, + c.flags.BoolVar(&c.kvDetails, "kvdetails", false, "Provides a detailed KV space usage breakdown for any KV data that's been stored.") - c.flags.IntVar(&c.depth, "depth", 2, - "Must be used with -detailed. The key prefix depth used to breakdown KV store data. Defaults to 2.") - c.flags.StringVar(&c.filter, "filter", "", - "Must be used with -detailed. Limits KV key breakdown using this prefix filter.") + c.flags.IntVar(&c.kvDepth, "kvdepth", 2, + "Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.") + c.flags.StringVar(&c.kvFilter, "kvfilter", "", + "Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.") c.flags.StringVar( &c.format, "format", @@ -270,7 +270,7 @@ func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { } func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) { - if c.detailed { + if c.kvDetails { if keyType != "KVS" { return } @@ -286,7 +286,7 @@ func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *Snapsho // check for whether a filter is specified. if it is, skip // any keys that don't match. - if len(c.filter) > 0 && !strings.HasPrefix(v.(string), c.filter) { + if len(c.kvFilter) > 0 && !strings.HasPrefix(v.(string), c.kvFilter) { break } @@ -294,8 +294,8 @@ func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *Snapsho // handle the situation where the key is shorter than // the specified depth. - actualDepth := c.depth - if c.depth > len(split) { + actualDepth := c.kvDepth + if c.kvDepth > len(split) { actualDepth = len(split) } prefix := strings.Join(split[0:actualDepth], "/") diff --git a/website/pages/commands/snapshot/inspect.mdx b/website/pages/commands/snapshot/inspect.mdx index c4ae783a4f..da36989963 100644 --- a/website/pages/commands/snapshot/inspect.mdx +++ b/website/pages/commands/snapshot/inspect.mdx @@ -109,7 +109,7 @@ more details about snapshot internals. #### Command Options -- `-detailed` - Optional, provides a space usage breakdown for any KV data stored in Consul. -- `-depth` - Must be used with `-detailed`. Used to adjust the grouping level of keys. Defaults to 2. -- `-filter` - Must be used with `-detailed`. Used to specify a key prefix that excludes keys that don't match. +- `-kvdetails` - Optional, provides a space usage breakdown for any KV data stored in Consul. +- `-kvdepth` - Can only be used with `-kvdetails`. Used to adjust the grouping level of keys. Defaults to 2. +- `-kvfilter` - Can only be used with `-kvdetails`. Used to specify a key prefix that excludes keys that don't match. - `-format` - Optional, allows from changing the output to JSON. Parameters accepted are "pretty" and "JSON". From 40a86818168c65eb0a64e33fe5ecfbd1765c9715 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 10:55:30 -0600 Subject: [PATCH 17/27] Allow omission of -kvdetails if another -kv* flag is set --- command/snapshot/inspect/snapshot_inspect.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 3ddd30300f..5c40c1c674 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -40,7 +40,7 @@ func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) c.flags.BoolVar(&c.kvDetails, "kvdetails", false, "Provides a detailed KV space usage breakdown for any KV data that's been stored.") - c.flags.IntVar(&c.kvDepth, "kvdepth", 2, + c.flags.IntVar(&c.kvDepth, "kvdepth", 0, "Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.") c.flags.StringVar(&c.kvFilter, "kvfilter", "", "Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.") @@ -103,6 +103,20 @@ func (c *cmd) Run(args []string) int { return 1 } + // automatically set kvDetails to true if a depth or filter + // is provided. this allows the user to omit the -kvdetails + // flag if they prefer. + if c.kvDepth != 0 || c.kvFilter != "" { + c.kvDetails = true + } + + // set the default depth if one wasn't specified with -kvdepth. + // this is used rather than the flag default to facilitate the + // above shortcut. + if c.kvDetails && c.kvDepth == 0 { + c.kvDepth = 2 + } + // Open the file. f, err := os.Open(file) if err != nil { From 7aebd179de6000072962c2b8fd8f6b59dff92811 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 11:18:21 -0600 Subject: [PATCH 18/27] Update tests for new flag names --- command/snapshot/inspect/formatter_test.go | 3 +- .../snapshot/inspect/snapshot_inspect_test.go | 36 ++++++++++---- ...estSnapshotInspectKVDetailsCommand.golden} | 0 ...spectKVDetailsCommandNoDetailsFlag.golden} | 2 +- ...napshotInspectKVDetailsDepthCommand.golden | 44 ++++++++++++++++++ ...InspectKVDetailsDepthFilterCommand.golden} | 2 +- ...{backup-with-kv.snap => backupWithKV.snap} | Bin command/snapshot/inspect/testdata/json.golden | 10 +++- .../snapshot/inspect/testdata/pretty.golden | 8 +++- 9 files changed, 90 insertions(+), 15 deletions(-) rename command/snapshot/inspect/testdata/{TestSnapshotInspectDetailedCommand.golden => TestSnapshotInspectKVDetailsCommand.golden} (100%) rename command/snapshot/inspect/testdata/{TestSnapshotInspectDetailedDepthCommand.golden => TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden} (100%) create mode 100644 command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden rename command/snapshot/inspect/testdata/{TestSnapshotInspectDetailedDepthFilterCommand.golden => TestSnapshotInspectKVDetailsDepthFilterCommand.golden} (100%) rename command/snapshot/inspect/testdata/{backup-with-kv.snap => backupWithKV.snap} (100%) diff --git a/command/snapshot/inspect/formatter_test.go b/command/snapshot/inspect/formatter_test.go index ec02a21a25..428995a27c 100644 --- a/command/snapshot/inspect/formatter_test.go +++ b/command/snapshot/inspect/formatter_test.go @@ -31,7 +31,6 @@ func TestFormat(t *testing.T) { TotalSize: 1, TotalSizeKV: 1, } - detailed := false formatters := map[string]Formatter{ "pretty": newPrettyFormatter(), @@ -41,7 +40,7 @@ func TestFormat(t *testing.T) { for fmtName, formatter := range formatters { t.Run(fmtName, func(t *testing.T) { - actual, err := formatter.Format(&info, detailed) + actual, err := formatter.Format(&info) require.NoError(t, err) gName := fmt.Sprintf("%s", fmtName) diff --git a/command/snapshot/inspect/snapshot_inspect_test.go b/command/snapshot/inspect/snapshot_inspect_test.go index 5e5c745533..db23dc9de5 100644 --- a/command/snapshot/inspect/snapshot_inspect_test.go +++ b/command/snapshot/inspect/snapshot_inspect_test.go @@ -96,14 +96,14 @@ func TestSnapshotInspectCommand(t *testing.T) { require.Equal(t, want, ui.OutputWriter.String()) } -func TestSnapshotInspectDetailedCommand(t *testing.T) { +func TestSnapshotInspectKVDetailsCommand(t *testing.T) { - filepath := "./testdata/backup-with-kv.snap" + filepath := "./testdata/backupWithKV.snap" // Inspect the snapshot ui := cli.NewMockUi() c := New(ui) - args := []string{"-detailed", filepath} + args := []string{"-kvdetails", filepath} code := c.Run(args) if code != 0 { @@ -114,14 +114,14 @@ func TestSnapshotInspectDetailedCommand(t *testing.T) { require.Equal(t, want, ui.OutputWriter.String()) } -func TestSnapshotInspectDetailedDepthCommand(t *testing.T) { +func TestSnapshotInspectKVDetailsDepthCommand(t *testing.T) { - filepath := "./testdata/backup-with-kv.snap" + filepath := "./testdata/backupWithKV.snap" // Inspect the snapshot ui := cli.NewMockUi() c := New(ui) - args := []string{"-detailed", "-depth", "3", filepath} + args := []string{"-kvdetails", "-kvdepth", "3", filepath} code := c.Run(args) if code != 0 { @@ -132,14 +132,32 @@ func TestSnapshotInspectDetailedDepthCommand(t *testing.T) { require.Equal(t, want, ui.OutputWriter.String()) } -func TestSnapshotInspectDetailedDepthFilterCommand(t *testing.T) { +func TestSnapshotInspectKVDetailsDepthFilterCommand(t *testing.T) { - filepath := "./testdata/backup-with-kv.snap" + filepath := "./testdata/backupWithKV.snap" // Inspect the snapshot ui := cli.NewMockUi() c := New(ui) - args := []string{"-detailed", "-depth", "3", "-filter", "vault/logical", filepath} + args := []string{"-kvdetails", "-kvdepth", "3", "-kvfilter", "vault/logical", filepath} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + want := golden(t, t.Name(), ui.OutputWriter.String()) + require.Equal(t, want, ui.OutputWriter.String()) +} + +func TestSnapshotInspectKVDetailsCommandNoDetailsFlag(t *testing.T) { + + filepath := "./testdata/backupWithKV.snap" + + // Inspect the snapshot + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-kvdepth", "3", filepath} code := c.Run(args) if code != 0 { diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden similarity index 100% rename from command/snapshot/inspect/testdata/TestSnapshotInspectDetailedCommand.golden rename to command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden similarity index 100% rename from command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden rename to command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden index 61e9108535..a962bd27bb 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden @@ -9,8 +9,8 @@ KVS 27 12.3KB Register 5 3.4KB Index 11 285B - Session 1 199B Autopilot 1 199B + Session 1 199B CoordinateBatchUpdate 1 166B Tombstone 2 146B FederationState 1 139B diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden new file mode 100644 index 0000000000..a962bd27bb --- /dev/null +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden @@ -0,0 +1,44 @@ + ID 2-12426-1604593650375 + Size 17228 + Index 12426 + Term 2 + Version 1 + + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB + + Key Name Count Size + ---- ---- ---- + vault/sys/policy 3 3.3KB + vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB + vault/core/leader 1 1.6KB + vault/sys/token 3 1KB + vault/core/mounts 1 675B + vault/core/wrapping 1 633B + vault/core/local-mounts 1 450B + vault/core/auth 1 423B + vault/core/cluster 2 388B + vault/core/keyring 1 320B + vault/core/master 1 237B + vault/core/seal-config 1 211B + vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B + vault/core/hsm 1 189B + vault/core/local-audit 1 185B + vault/core/local-auth 1 183B + vault/core/audit 1 179B + vault/core/lock 1 170B + vault/core/shamir-kek 1 159B + vault/sys/counters 1 155B + ---- ---- ---- + Total 12.3KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden similarity index 100% rename from command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden rename to command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden index a72fb99b52..881a10cfcd 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectDetailedDepthFilterCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden @@ -9,8 +9,8 @@ KVS 27 12.3KB Register 5 3.4KB Index 11 285B - Session 1 199B Autopilot 1 199B + Session 1 199B CoordinateBatchUpdate 1 166B Tombstone 2 146B FederationState 1 139B diff --git a/command/snapshot/inspect/testdata/backup-with-kv.snap b/command/snapshot/inspect/testdata/backupWithKV.snap similarity index 100% rename from command/snapshot/inspect/testdata/backup-with-kv.snap rename to command/snapshot/inspect/testdata/backupWithKV.snap diff --git a/command/snapshot/inspect/testdata/json.golden b/command/snapshot/inspect/testdata/json.golden index 42c3ba40b4..261d3d8b27 100644 --- a/command/snapshot/inspect/testdata/json.golden +++ b/command/snapshot/inspect/testdata/json.golden @@ -13,5 +13,13 @@ "Count": 2 } ], - "TotalSize": 1 + "StatsKV": [ + { + "Name": "msgKV", + "Sum": 1, + "Count": 2 + } + ], + "TotalSize": 1, + "TotalSizeKV": 1 } \ No newline at end of file diff --git a/command/snapshot/inspect/testdata/pretty.golden b/command/snapshot/inspect/testdata/pretty.golden index 7a2c5749b3..2a8d028ff4 100644 --- a/command/snapshot/inspect/testdata/pretty.golden +++ b/command/snapshot/inspect/testdata/pretty.golden @@ -8,4 +8,10 @@ ---- ---- ---- msg 2 1B ---- ---- ---- - Total 1B \ No newline at end of file + Total 1B + + Key Name Count Size + ---- ---- ---- + msgKV 2 1B + ---- ---- ---- + Total 1B \ No newline at end of file From 6957056911759226bbe0963767f4c29718cdbe5f Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 11:22:45 -0600 Subject: [PATCH 19/27] Missed a spot with old params in docs --- website/pages/commands/snapshot/inspect.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/pages/commands/snapshot/inspect.mdx b/website/pages/commands/snapshot/inspect.mdx index da36989963..1f3f1c5273 100644 --- a/website/pages/commands/snapshot/inspect.mdx +++ b/website/pages/commands/snapshot/inspect.mdx @@ -62,7 +62,7 @@ $ consul snapshot inspect backup.snap To get more details for a snapshot inspection from "backup.snap": ```shell-session -$ consul snapshot inspect -detailed -depth 3 -filter vault/core backup.snap +$ consul snapshot inspect -kvdetails -kvdepth 3 -kvfilter vault/core backup.snap ID 2-12426-1604593650375 Size 17228 Index 12426 From ea804e57b44d382804d2ca0dfecb6854dc504d05 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 16:39:12 -0600 Subject: [PATCH 20/27] Check for nil rather than length --- command/snapshot/inspect/formatter.go | 2 +- command/snapshot/inspect/snapshot_inspect.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index bd45501ab4..c8c308b5f9 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -57,7 +57,7 @@ func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) { fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) - if len(info.StatsKV) > 0 { + if info.StatsKV != nil { fmt.Fprintf(tw, "\n") fmt.Fprintln(tw, "\n Key Name\tCount\tSize\t") fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 5c40c1c674..83216cc4d9 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -199,7 +199,7 @@ func generateStats(info SnapshotInfo) []typeStats { } func generateKVStats(info SnapshotInfo) []typeStats { - if len(info.StatsKV) > 0 { + if info.StatsKV != nil { ks := make([]typeStats, 0, len(info.StatsKV)) for _, s := range info.StatsKV { From 99ec1febcc4fdb091621b147ac49a9498958f844 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 16:39:40 -0600 Subject: [PATCH 21/27] Move kvDetails default logic --- command/snapshot/inspect/snapshot_inspect.go | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 83216cc4d9..4269883013 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -103,20 +103,6 @@ func (c *cmd) Run(args []string) int { return 1 } - // automatically set kvDetails to true if a depth or filter - // is provided. this allows the user to omit the -kvdetails - // flag if they prefer. - if c.kvDepth != 0 || c.kvFilter != "" { - c.kvDetails = true - } - - // set the default depth if one wasn't specified with -kvdepth. - // this is used rather than the flag default to facilitate the - // above shortcut. - if c.kvDetails && c.kvDepth == 0 { - c.kvDepth = 2 - } - // Open the file. f, err := os.Open(file) if err != nil { @@ -284,6 +270,20 @@ func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { } func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) { + // automatically set kvDetails to true if a depth or filter + // is provided. this allows the user to omit the -kvdetails + // flag if they prefer. + if c.kvDepth != 0 || c.kvFilter != "" { + c.kvDetails = true + } + + // set the default depth if one wasn't specified with -kvdepth. + // this is used rather than the flag default to facilitate the + // above shortcut. + if c.kvDetails && c.kvDepth == 0 { + c.kvDepth = 2 + } + if c.kvDetails { if keyType != "KVS" { return From 491f2f0a57ae752938f7963b3898b46ad9f6f9f0 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 16:40:12 -0600 Subject: [PATCH 22/27] Add more func comments --- command/snapshot/inspect/snapshot_inspect.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 4269883013..61a641cf5b 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -172,6 +172,8 @@ type typeStats struct { Count int } +// Generate the stats for the output struct that's +// used to produce the printed output the user sees. func generateStats(info SnapshotInfo) []typeStats { ss := make([]typeStats, 0, len(info.Stats)) @@ -184,6 +186,8 @@ func generateStats(info SnapshotInfo) []typeStats { return ss } +// Generate the KV stats for the output struct that's +// used to produce the printed output the user sees. func generateKVStats(info SnapshotInfo) []typeStats { if info.StatsKV != nil { ks := make([]typeStats, 0, len(info.StatsKV)) @@ -269,6 +273,8 @@ func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { } +// kvEnhance populates the struct with all of the snapshot's +// size information for KV data stored in it func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) { // automatically set kvDetails to true if a depth or filter // is provided. this allows the user to omit the -kvdetails From 48a358df1bd2b41cfec9fb19ec65c66324aa6ac5 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Tue, 10 Nov 2020 16:40:23 -0600 Subject: [PATCH 23/27] Remove trailing tabs from output --- command/snapshot/inspect/formatter.go | 20 +++--- .../TestSnapshotInspectCommand.golden | 24 +++---- .../TestSnapshotInspectEnhanceCommand.golden | 22 +++--- ...TestSnapshotInspectKVDetailsCommand.golden | 38 +++++----- ...nspectKVDetailsCommandNoDetailsFlag.golden | 72 +++++++++---------- ...napshotInspectKVDetailsDepthCommand.golden | 72 +++++++++---------- ...tInspectKVDetailsDepthFilterCommand.golden | 36 +++++----- .../snapshot/inspect/testdata/pretty.golden | 18 ++--- 8 files changed, 151 insertions(+), 151 deletions(-) diff --git a/command/snapshot/inspect/formatter.go b/command/snapshot/inspect/formatter.go index c8c308b5f9..1258fb8502 100644 --- a/command/snapshot/inspect/formatter.go +++ b/command/snapshot/inspect/formatter.go @@ -48,25 +48,25 @@ func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) { fmt.Fprintf(tw, "\n Term\t%d", info.Meta.Term) fmt.Fprintf(tw, "\n Version\t%d", info.Meta.Version) fmt.Fprintf(tw, "\n") - fmt.Fprintln(tw, "\n Type\tCount\tSize\t") - fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") + fmt.Fprintln(tw, "\n Type\tCount\tSize") + fmt.Fprintf(tw, " %s\t%s\t%s", "----", "----", "----") // For each different type generate new output for _, s := range info.Stats { - fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) + fmt.Fprintf(tw, "\n %s\t%d\t%s", s.Name, s.Count, ByteSize(uint64(s.Sum))) } - fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") - fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize))) + fmt.Fprintf(tw, "\n %s\t%s\t%s", "----", "----", "----") + fmt.Fprintf(tw, "\n Total\t\t%s", ByteSize(uint64(info.TotalSize))) if info.StatsKV != nil { fmt.Fprintf(tw, "\n") - fmt.Fprintln(tw, "\n Key Name\tCount\tSize\t") - fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----") + fmt.Fprintln(tw, "\n Key Name\tCount\tSize") + fmt.Fprintf(tw, " %s\t%s\t%s", "----", "----", "----") // For each different type generate new output for _, s := range info.StatsKV { - fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum))) + fmt.Fprintf(tw, "\n %s\t%d\t%s", s.Name, s.Count, ByteSize(uint64(s.Sum))) } - fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----") - fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSizeKV))) + fmt.Fprintf(tw, "\n %s\t%s\t%s", "----", "----", "----") + fmt.Fprintf(tw, "\n Total\t\t%s", ByteSize(uint64(info.TotalSizeKV))) } if err := tw.Flush(); err != nil { diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden index 00b9da4066..fc5af33af8 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectCommand.golden @@ -4,16 +4,16 @@ Term 2 Version 1 - Type Count Size - ---- ---- ---- - Register 3 1.7KB - ConnectCA 1 1.2KB - ConnectCAProviderState 1 1.1KB - Index 12 344B - Autopilot 1 199B - ConnectCAConfig 1 197B - FederationState 1 139B - SystemMetadata 1 68B - ChunkingState 1 12B - ---- ---- ---- + Type Count Size + ---- ---- ---- + Register 3 1.7KB + ConnectCA 1 1.2KB + ConnectCAProviderState 1 1.1KB + Index 12 344B + Autopilot 1 199B + ConnectCAConfig 1 197B + FederationState 1 139B + SystemMetadata 1 68B + ChunkingState 1 12B + ---- ---- ---- Total 5KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden index 9867d8e5aa..d85f58ab44 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectEnhanceCommand.golden @@ -4,15 +4,15 @@ Term 2 Version 1 - Type Count Size - ---- ---- ---- - Register 3 1.8KB - ConnectCA 1 1.2KB - ConnectCAProviderState 1 1.1KB - Index 11 313B - ConnectCAConfig 1 247B - Autopilot 1 199B - SystemMetadata 1 68B - ChunkingState 1 12B - ---- ---- ---- + Type Count Size + ---- ---- ---- + Register 3 1.8KB + ConnectCA 1 1.2KB + ConnectCAProviderState 1 1.1KB + Index 11 313B + ConnectCAConfig 1 247B + Autopilot 1 199B + SystemMetadata 1 68B + ChunkingState 1 12B + ---- ---- ---- Total 5KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden index b16f6efd39..2986dcf923 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden @@ -4,24 +4,24 @@ Term 2 Version 1 - Type Count Size - ---- ---- ---- - KVS 27 12.3KB - Register 5 3.4KB - Index 11 285B - Autopilot 1 199B - Session 1 199B - CoordinateBatchUpdate 1 166B - Tombstone 2 146B - FederationState 1 139B - ChunkingState 1 12B - ---- ---- ---- - Total 16.8KB + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB - Key Name Count Size - ---- ---- ---- - vault/core 16 5.9KB - vault/sys 7 4.4KB - vault/logical 4 2KB - ---- ---- ---- + Key Name Count Size + ---- ---- ---- + vault/core 16 5.9KB + vault/sys 7 4.4KB + vault/logical 4 2KB + ---- ---- ---- Total 12.3KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden index a962bd27bb..1ce433c3a3 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden @@ -4,41 +4,41 @@ Term 2 Version 1 - Type Count Size - ---- ---- ---- - KVS 27 12.3KB - Register 5 3.4KB - Index 11 285B - Autopilot 1 199B - Session 1 199B - CoordinateBatchUpdate 1 166B - Tombstone 2 146B - FederationState 1 139B - ChunkingState 1 12B - ---- ---- ---- - Total 16.8KB + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB - Key Name Count Size - ---- ---- ---- - vault/sys/policy 3 3.3KB - vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB - vault/core/leader 1 1.6KB - vault/sys/token 3 1KB - vault/core/mounts 1 675B - vault/core/wrapping 1 633B - vault/core/local-mounts 1 450B - vault/core/auth 1 423B - vault/core/cluster 2 388B - vault/core/keyring 1 320B - vault/core/master 1 237B - vault/core/seal-config 1 211B - vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B - vault/core/hsm 1 189B - vault/core/local-audit 1 185B - vault/core/local-auth 1 183B - vault/core/audit 1 179B - vault/core/lock 1 170B - vault/core/shamir-kek 1 159B - vault/sys/counters 1 155B - ---- ---- ---- + Key Name Count Size + ---- ---- ---- + vault/sys/policy 3 3.3KB + vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB + vault/core/leader 1 1.6KB + vault/sys/token 3 1KB + vault/core/mounts 1 675B + vault/core/wrapping 1 633B + vault/core/local-mounts 1 450B + vault/core/auth 1 423B + vault/core/cluster 2 388B + vault/core/keyring 1 320B + vault/core/master 1 237B + vault/core/seal-config 1 211B + vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B + vault/core/hsm 1 189B + vault/core/local-audit 1 185B + vault/core/local-auth 1 183B + vault/core/audit 1 179B + vault/core/lock 1 170B + vault/core/shamir-kek 1 159B + vault/sys/counters 1 155B + ---- ---- ---- Total 12.3KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden index a962bd27bb..1ce433c3a3 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden @@ -4,41 +4,41 @@ Term 2 Version 1 - Type Count Size - ---- ---- ---- - KVS 27 12.3KB - Register 5 3.4KB - Index 11 285B - Autopilot 1 199B - Session 1 199B - CoordinateBatchUpdate 1 166B - Tombstone 2 146B - FederationState 1 139B - ChunkingState 1 12B - ---- ---- ---- - Total 16.8KB + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB - Key Name Count Size - ---- ---- ---- - vault/sys/policy 3 3.3KB - vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB - vault/core/leader 1 1.6KB - vault/sys/token 3 1KB - vault/core/mounts 1 675B - vault/core/wrapping 1 633B - vault/core/local-mounts 1 450B - vault/core/auth 1 423B - vault/core/cluster 2 388B - vault/core/keyring 1 320B - vault/core/master 1 237B - vault/core/seal-config 1 211B - vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B - vault/core/hsm 1 189B - vault/core/local-audit 1 185B - vault/core/local-auth 1 183B - vault/core/audit 1 179B - vault/core/lock 1 170B - vault/core/shamir-kek 1 159B - vault/sys/counters 1 155B - ---- ---- ---- + Key Name Count Size + ---- ---- ---- + vault/sys/policy 3 3.3KB + vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB + vault/core/leader 1 1.6KB + vault/sys/token 3 1KB + vault/core/mounts 1 675B + vault/core/wrapping 1 633B + vault/core/local-mounts 1 450B + vault/core/auth 1 423B + vault/core/cluster 2 388B + vault/core/keyring 1 320B + vault/core/master 1 237B + vault/core/seal-config 1 211B + vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B + vault/core/hsm 1 189B + vault/core/local-audit 1 185B + vault/core/local-auth 1 183B + vault/core/audit 1 179B + vault/core/lock 1 170B + vault/core/shamir-kek 1 159B + vault/sys/counters 1 155B + ---- ---- ---- Total 12.3KB diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden index 881a10cfcd..7e048cc6a8 100644 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden +++ b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden @@ -4,23 +4,23 @@ Term 2 Version 1 - Type Count Size - ---- ---- ---- - KVS 27 12.3KB - Register 5 3.4KB - Index 11 285B - Autopilot 1 199B - Session 1 199B - CoordinateBatchUpdate 1 166B - Tombstone 2 146B - FederationState 1 139B - ChunkingState 1 12B - ---- ---- ---- - Total 16.8KB + Type Count Size + ---- ---- ---- + KVS 27 12.3KB + Register 5 3.4KB + Index 11 285B + Autopilot 1 199B + Session 1 199B + CoordinateBatchUpdate 1 166B + Tombstone 2 146B + FederationState 1 139B + ChunkingState 1 12B + ---- ---- ---- + Total 16.8KB - Key Name Count Size - ---- ---- ---- - vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB - vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B - ---- ---- ---- + Key Name Count Size + ---- ---- ---- + vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB + vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B + ---- ---- ---- Total 2KB diff --git a/command/snapshot/inspect/testdata/pretty.golden b/command/snapshot/inspect/testdata/pretty.golden index 2a8d028ff4..a710fb6f98 100644 --- a/command/snapshot/inspect/testdata/pretty.golden +++ b/command/snapshot/inspect/testdata/pretty.golden @@ -4,14 +4,14 @@ Term 4 Version 1 - Type Count Size - ---- ---- ---- - msg 2 1B - ---- ---- ---- - Total 1B + Type Count Size + ---- ---- ---- + msg 2 1B + ---- ---- ---- + Total 1B - Key Name Count Size - ---- ---- ---- - msgKV 2 1B - ---- ---- ---- + Key Name Count Size + ---- ---- ---- + msgKV 2 1B + ---- ---- ---- Total 1B \ No newline at end of file From 5b4c8571d281dee965f482b0aab11673f01e3eeb Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 11 Nov 2020 09:56:26 -0600 Subject: [PATCH 24/27] Add changelog entry --- .changelog/9098.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/9098.txt diff --git a/.changelog/9098.txt b/.changelog/9098.txt new file mode 100644 index 0000000000..7eab768344 --- /dev/null +++ b/.changelog/9098.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: snapshot inspect command provides KV space usage breakdown +``` From 0d485594167a9bf522ac8bae11adf621c5871cb7 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 11 Nov 2020 10:44:55 -0600 Subject: [PATCH 25/27] Length check is required here If one isn't included, then the nil check in the formatter never fails due to an empty slice being passed in, which causes the kv output to always get printed. --- command/snapshot/inspect/snapshot_inspect.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 61a641cf5b..468250fdab 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -189,8 +189,9 @@ func generateStats(info SnapshotInfo) []typeStats { // Generate the KV stats for the output struct that's // used to produce the printed output the user sees. func generateKVStats(info SnapshotInfo) []typeStats { - if info.StatsKV != nil { - ks := make([]typeStats, 0, len(info.StatsKV)) + kvLen := len(info.StatsKV) + if kvLen > 0 { + ks := make([]typeStats, 0, kvLen) for _, s := range info.StatsKV { ks = append(ks, s) From 94689b6ecd67bad0a26a8e96f0894c0d8f49ea6f Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 11 Nov 2020 10:46:51 -0600 Subject: [PATCH 26/27] Back out implicit -kvdetails functionality --- command/snapshot/inspect/snapshot_inspect.go | 16 +------ .../snapshot/inspect/snapshot_inspect_test.go | 18 -------- ...nspectKVDetailsCommandNoDetailsFlag.golden | 44 ------------------- 3 files changed, 1 insertion(+), 77 deletions(-) delete mode 100644 command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index 468250fdab..ff4b8f0594 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -40,7 +40,7 @@ func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) c.flags.BoolVar(&c.kvDetails, "kvdetails", false, "Provides a detailed KV space usage breakdown for any KV data that's been stored.") - c.flags.IntVar(&c.kvDepth, "kvdepth", 0, + c.flags.IntVar(&c.kvDepth, "kvdepth", 2, "Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.") c.flags.StringVar(&c.kvFilter, "kvfilter", "", "Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.") @@ -277,20 +277,6 @@ func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) { // kvEnhance populates the struct with all of the snapshot's // size information for KV data stored in it func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) { - // automatically set kvDetails to true if a depth or filter - // is provided. this allows the user to omit the -kvdetails - // flag if they prefer. - if c.kvDepth != 0 || c.kvFilter != "" { - c.kvDetails = true - } - - // set the default depth if one wasn't specified with -kvdepth. - // this is used rather than the flag default to facilitate the - // above shortcut. - if c.kvDetails && c.kvDepth == 0 { - c.kvDepth = 2 - } - if c.kvDetails { if keyType != "KVS" { return diff --git a/command/snapshot/inspect/snapshot_inspect_test.go b/command/snapshot/inspect/snapshot_inspect_test.go index db23dc9de5..fbfc60feb0 100644 --- a/command/snapshot/inspect/snapshot_inspect_test.go +++ b/command/snapshot/inspect/snapshot_inspect_test.go @@ -149,21 +149,3 @@ func TestSnapshotInspectKVDetailsDepthFilterCommand(t *testing.T) { want := golden(t, t.Name(), ui.OutputWriter.String()) require.Equal(t, want, ui.OutputWriter.String()) } - -func TestSnapshotInspectKVDetailsCommandNoDetailsFlag(t *testing.T) { - - filepath := "./testdata/backupWithKV.snap" - - // Inspect the snapshot - ui := cli.NewMockUi() - c := New(ui) - args := []string{"-kvdepth", "3", filepath} - - code := c.Run(args) - if code != 0 { - t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) - } - - want := golden(t, t.Name(), ui.OutputWriter.String()) - require.Equal(t, want, ui.OutputWriter.String()) -} diff --git a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden b/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden deleted file mode 100644 index 1ce433c3a3..0000000000 --- a/command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommandNoDetailsFlag.golden +++ /dev/null @@ -1,44 +0,0 @@ - ID 2-12426-1604593650375 - Size 17228 - Index 12426 - Term 2 - Version 1 - - Type Count Size - ---- ---- ---- - KVS 27 12.3KB - Register 5 3.4KB - Index 11 285B - Autopilot 1 199B - Session 1 199B - CoordinateBatchUpdate 1 166B - Tombstone 2 146B - FederationState 1 139B - ChunkingState 1 12B - ---- ---- ---- - Total 16.8KB - - Key Name Count Size - ---- ---- ---- - vault/sys/policy 3 3.3KB - vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9 3 1.8KB - vault/core/leader 1 1.6KB - vault/sys/token 3 1KB - vault/core/mounts 1 675B - vault/core/wrapping 1 633B - vault/core/local-mounts 1 450B - vault/core/auth 1 423B - vault/core/cluster 2 388B - vault/core/keyring 1 320B - vault/core/master 1 237B - vault/core/seal-config 1 211B - vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0 1 210B - vault/core/hsm 1 189B - vault/core/local-audit 1 185B - vault/core/local-auth 1 183B - vault/core/audit 1 179B - vault/core/lock 1 170B - vault/core/shamir-kek 1 159B - vault/sys/counters 1 155B - ---- ---- ---- - Total 12.3KB From 182333b645e746b286c8f4eb895e26c4b0139463 Mon Sep 17 00:00:00 2001 From: Joel Watson Date: Wed, 11 Nov 2020 11:33:38 -0600 Subject: [PATCH 27/27] Fix some minor wording issues --- .changelog/9098.txt | 2 +- command/snapshot/inspect/snapshot_inspect.go | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.changelog/9098.txt b/.changelog/9098.txt index 7eab768344..2963aa754f 100644 --- a/.changelog/9098.txt +++ b/.changelog/9098.txt @@ -1,3 +1,3 @@ ```release-note:feature -cli: snapshot inspect command provides KV space usage breakdown +cli: snapshot inspect command provides KV usage breakdown ``` diff --git a/command/snapshot/inspect/snapshot_inspect.go b/command/snapshot/inspect/snapshot_inspect.go index ff4b8f0594..e0bbd648c2 100644 --- a/command/snapshot/inspect/snapshot_inspect.go +++ b/command/snapshot/inspect/snapshot_inspect.go @@ -172,8 +172,8 @@ type typeStats struct { Count int } -// Generate the stats for the output struct that's -// used to produce the printed output the user sees. +// generateStats formats the stats for the output struct +// that's used to produce the printed output the user sees. func generateStats(info SnapshotInfo) []typeStats { ss := make([]typeStats, 0, len(info.Stats)) @@ -186,8 +186,9 @@ func generateStats(info SnapshotInfo) []typeStats { return ss } -// Generate the KV stats for the output struct that's -// used to produce the printed output the user sees. +// generateKVStats reformats the KV stats to work with +// the output struct that's used to produce the printed +// output the user sees. func generateKVStats(info SnapshotInfo) []typeStats { kvLen := len(info.StatsKV) if kvLen > 0 { @@ -205,7 +206,8 @@ func generateKVStats(info SnapshotInfo) []typeStats { return nil } -// Sort the stat slice +// sortTypeStats sorts the stat slice by size and then +// alphabetically in the case the size is identical func sortTypeStats(stats []typeStats) []typeStats { sort.Slice(stats, func(i, j int) bool { // sort alphabetically if size is equal