command: ensure that the json form of config entries can be submitted with 'consul config write' (#6290)

The json decoder inside of the HCLv1 hcl.Decode function behaves
unexpectedly when decoding generically into a map[string]interface{} as
is done for 'consul config write' pre-submit decoding.

This results in some subtle (service-router Match and Destinations being
separated) and some not so subtle (service-resolver subsets and failover
panic if multiple subsets are referenced) bugs when subsequently passed
through mapstructure to finish decoding.

Given that HCLv1 is basically frozen and the HCL part of it is fine
instead of trying to figure out what the underlying bug is in the json
decoder for our purposes just sniff the byte slice and selectively use
the stdlib json decoder for JSON and hcl decoder for HCL.

command/config/write/config_write.go

@@ -11,7 +11,6 @@ import (
 	"github.com/hashicorp/consul/command/helpers"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/go-multierror"
-	"github.com/hashicorp/hcl"
 	"github.com/mitchellh/cli"
 	"github.com/mitchellh/mapstructure"
 )
@@ -102,7 +101,7 @@ func (c *cmd) Run(args []string) int {
 func parseConfigEntry(data string) (api.ConfigEntry, error) {
 	// parse the data
 	var raw map[string]interface{}
-	if err := hcl.Decode(&raw, data); err != nil {
+	if err := hclDecode(&raw, data); err != nil {
 		return nil, fmt.Errorf("Failed to decode config entry input: %v", err)
 	}
 

command/config/write/config_write_test.go

@@ -111,13 +111,13 @@ func TestConfigWrite(t *testing.T) {
 func TestParseConfigEntry(t *testing.T) {
 	t.Parallel()
 	for _, tc := range []struct {
-		name      string
+		name             string
-		camel     string
+		camel, camelJSON string
-		snake     string
+		snake, snakeJSON string
-		expect    api.ConfigEntry
+		expect           api.ConfigEntry
-		expectErr string
+		expectJSON       api.ConfigEntry
+		expectErr        string
 	}{
-		// TODO(rb): test json?
 		{
 			name: "proxy-defaults: extra fields or typo",
 			snake: `
@@ -134,6 +134,24 @@ func TestParseConfigEntry(t *testing.T) {
 				  "foo" = 19
 				}
 			`,
+			snakeJSON: `
+			{
+				"kind": "proxy-defaults",
+				"name": "main",
+				"cornfig": {
+					"foo": 19
+				}
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "proxy-defaults",
+				"Name": "main",
+				"Cornfig": {
+					"foo": 19
+				}
+			}
+			`,
 			expectErr: `invalid config key "cornfig"`,
 		},
 		{
@@ -166,6 +184,38 @@ func TestParseConfigEntry(t *testing.T) {
 					Mode = "remote"
 				}
 			`,
+			snakeJSON: `
+			{
+				"kind": "proxy-defaults",
+				"name": "main",
+				"config": {
+					"foo": 19,
+					"bar": "abc",
+					"moreconfig": {
+						"moar": "config"
+					}
+				},
+				"mesh_gateway": {
+					"mode": "remote"
+				}
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "proxy-defaults",
+				"Name": "main",
+				"Config": {
+					"foo": 19,
+					"bar": "abc",
+					"moreconfig": {
+						"moar": "config"
+					}
+				},
+				"MeshGateway": {
+					"Mode": "remote"
+				}
+			}
+			`,
 			expect: &api.ProxyConfigEntry{
 				Kind: "proxy-defaults",
 				Name: "main",
@@ -180,6 +230,20 @@ func TestParseConfigEntry(t *testing.T) {
 					Mode: api.MeshGatewayModeRemote,
 				},
 			},
+			expectJSON: &api.ProxyConfigEntry{
+				Kind: "proxy-defaults",
+				Name: "main",
+				Config: map[string]interface{}{
+					"foo": float64(19), // json decoding gives float64 instead of int here
+					"bar": "abc",
+					"moreconfig": map[string]interface{}{
+						"moar": "config",
+					},
+				},
+				MeshGateway: api.MeshGatewayConfig{
+					Mode: api.MeshGatewayModeRemote,
+				},
+			},
 		},
 		{
 			name: "service-defaults",
@@ -199,6 +263,26 @@ func TestParseConfigEntry(t *testing.T) {
 					Mode = "remote"
 				}
 			`,
+			snakeJSON: `
+			{
+				"kind": "service-defaults",
+				"name": "main",
+				"protocol": "http",
+				"mesh_gateway": {
+					"mode": "remote"
+				}
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "service-defaults",
+				"Name": "main",
+				"Protocol": "http",
+				"MeshGateway": {
+					"Mode": "remote"
+				}
+			}
+			`,
 			expect: &api.ServiceConfigEntry{
 				Kind:     "service-defaults",
 				Name:     "main",
@@ -368,6 +452,180 @@ func TestParseConfigEntry(t *testing.T) {
 					},
 				]
 			`,
+			snakeJSON: `
+			{
+				"kind": "service-router",
+				"name": "main",
+				"routes": [
+					{
+						"match": {
+							"http": {
+								"path_exact": "/foo",
+								"header": [
+									{
+										"name": "debug1",
+										"present": true
+									},
+									{
+										"name": "debug2",
+										"present": false,
+										"invert": true
+									},
+									{
+										"name": "debug3",
+										"exact": "1"
+									},
+									{
+										"name": "debug4",
+										"prefix": "aaa"
+									},
+									{
+										"name": "debug5",
+										"suffix": "bbb"
+									},
+									{
+										"name": "debug6",
+										"regex": "a.*z"
+									}
+								]
+							}
+						},
+						"destination": {
+							"service": "carrot",
+							"service_subset": "kale",
+							"namespace": "leek",
+							"prefix_rewrite": "/alternate",
+							"request_timeout": "99s",
+							"num_retries": 12345,
+							"retry_on_connect_failure": true,
+							"retry_on_status_codes": [
+								401,
+								209
+							]
+						}
+					},
+					{
+						"match": {
+							"http": {
+								"path_prefix": "/foo",
+								"methods": [
+									"GET",
+									"DELETE"
+								],
+								"query_param": [
+									{
+										"name": "hack1",
+										"present": true
+									},
+									{
+										"name": "hack2",
+										"exact": "1"
+									},
+									{
+										"name": "hack3",
+										"regex": "a.*z"
+									}
+								]
+							}
+						}
+					},
+					{
+						"match": {
+							"http": {
+								"path_regex": "/foo"
+							}
+						}
+					}
+				]
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "service-router",
+				"Name": "main",
+				"Routes": [
+					{
+						"Match": {
+							"HTTP": {
+								"PathExact": "/foo",
+								"Header": [
+									{
+										"Name": "debug1",
+										"Present": true
+									},
+									{
+										"Name": "debug2",
+										"Present": false,
+										"Invert": true
+									},
+									{
+										"Name": "debug3",
+										"Exact": "1"
+									},
+									{
+										"Name": "debug4",
+										"Prefix": "aaa"
+									},
+									{
+										"Name": "debug5",
+										"Suffix": "bbb"
+									},
+									{
+										"Name": "debug6",
+										"Regex": "a.*z"
+									}
+								]
+							}
+						},
+						"Destination": {
+							"Service": "carrot",
+							"ServiceSubset": "kale",
+							"Namespace": "leek",
+							"PrefixRewrite": "/alternate",
+							"RequestTimeout": "99s",
+							"NumRetries": 12345,
+							"RetryOnConnectFailure": true,
+							"RetryOnStatusCodes": [
+								401,
+								209
+							]
+						}
+					},
+					{
+						"Match": {
+							"HTTP": {
+								"PathPrefix": "/foo",
+								"Methods": [
+									"GET",
+									"DELETE"
+								],
+								"QueryParam": [
+									{
+										"Name": "hack1",
+										"Present": true
+									},
+									{
+										"Name": "hack2",
+										"Exact": "1"
+									},
+									{
+										"Name": "hack3",
+										"Regex": "a.*z"
+									}
+								]
+							}
+						}
+					},
+					{
+						"Match": {
+							"HTTP": {
+								"PathRegex": "/foo"
+							}
+						}
+					}
+				]
+			}
+			`,
 			expect: &api.ServiceRouterConfigEntry{
 				Kind: "service-router",
 				Name: "main",
@@ -455,9 +713,13 @@ func TestParseConfigEntry(t *testing.T) {
 				name = "main"
 				splits = [
 				  {
-					weight        = 99.1
+					weight        = 97.1
 					service_subset = "v1"
 				  },
+				  {
+					weight        = 2
+					service_subset = "v2"
+				  },
 				  {
 					weight    = 0.9
 					service   = "other"
@@ -470,9 +732,13 @@ func TestParseConfigEntry(t *testing.T) {
 				Name = "main"
 				Splits = [
 				  {
-					Weight        = 99.1
+					Weight        = 97.1
 					ServiceSubset = "v1"
 				  },
+				  {
+					Weight        = 2,
+					ServiceSubset = "v2"
+				  },
 				  {
 					Weight    = 0.9
 					Service   = "other"
@@ -480,14 +746,60 @@ func TestParseConfigEntry(t *testing.T) {
 				  },
 				]
 			`,
+			snakeJSON: `
+			{
+				"kind": "service-splitter",
+				"name": "main",
+				"splits": [
+					{
+						"weight": 97.1,
+						"service_subset": "v1"
+					},
+					{
+						"weight": 2,
+						"service_subset": "v2"
+					},
+					{
+						"weight": 0.9,
+						"service": "other",
+						"namespace": "alt"
+					}
+				]
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "service-splitter",
+				"Name": "main",
+				"Splits": [
+					{
+						"Weight": 97.1,
+						"ServiceSubset": "v1"
+					},
+					{
+						"Weight": 2,
+						"ServiceSubset": "v2"
+					},
+					{
+						"Weight": 0.9,
+						"Service": "other",
+						"Namespace": "alt"
+					}
+				]
+			}
+			`,
 			expect: &api.ServiceSplitterConfigEntry{
 				Kind: api.ServiceSplitter,
 				Name: "main",
 				Splits: []api.ServiceSplit{
 					{
-						Weight:        99.1,
+						Weight:        97.1,
 						ServiceSubset: "v1",
 					},
+					{
+						Weight:        2,
+						ServiceSubset: "v2",
+					},
 					{
 						Weight:    0.9,
 						Service:   "other",
@@ -548,6 +860,72 @@ func TestParseConfigEntry(t *testing.T) {
 						Datacenters = ["dc7"]
 					}
 				}`,
+			snakeJSON: `
+			{
+				"kind": "service-resolver",
+				"name": "main",
+				"default_subset": "v1",
+				"connect_timeout": "15s",
+				"subsets": {
+					"v1": {
+						"filter": "Service.Meta.version == v1"
+					},
+					"v2": {
+						"filter": "Service.Meta.version == v2",
+						"only_passing": true
+					}
+				},
+				"failover": {
+					"v2": {
+						"service": "failcopy",
+						"service_subset": "sure",
+						"namespace": "neighbor",
+						"datacenters": [
+							"dc5",
+							"dc14"
+						]
+					},
+					"*": {
+						"datacenters": [
+							"dc7"
+						]
+					}
+				}
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "service-resolver",
+				"Name": "main",
+				"DefaultSubset": "v1",
+				"ConnectTimeout": "15s",
+				"Subsets": {
+					"v1": {
+						"Filter": "Service.Meta.version == v1"
+					},
+					"v2": {
+						"Filter": "Service.Meta.version == v2",
+						"OnlyPassing": true
+					}
+				},
+				"Failover": {
+					"v2": {
+						"Service": "failcopy",
+						"ServiceSubset": "sure",
+						"Namespace": "neighbor",
+						"Datacenters": [
+							"dc5",
+							"dc14"
+						]
+					},
+					"*": {
+						"Datacenters": [
+							"dc7"
+						]
+					}
+				}
+			}
+			`,
 			expect: &api.ServiceResolverConfigEntry{
 				Kind:           "service-resolver",
 				Name:           "main",
@@ -597,6 +975,30 @@ func TestParseConfigEntry(t *testing.T) {
 					Datacenter = "dc9"
 				}
 			`,
+			snakeJSON: `
+			{
+				"kind": "service-resolver",
+				"name": "main",
+				"redirect": {
+					"service": "other",
+					"service_subset": "backup",
+					"namespace": "alt",
+					"datacenter": "dc9"
+				}
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "service-resolver",
+				"Name": "main",
+				"Redirect": {
+					"Service": "other",
+					"ServiceSubset": "backup",
+					"Namespace": "alt",
+					"Datacenter": "dc9"
+				}
+			}
+			`,
 			expect: &api.ServiceResolverConfigEntry{
 				Kind: "service-resolver",
 				Name: "main",
@@ -618,6 +1020,18 @@ func TestParseConfigEntry(t *testing.T) {
 				Kind = "service-resolver"
 				Name = "main"
 			`,
+			snakeJSON: `
+			{
+				"kind": "service-resolver",
+				"name": "main"
+			}
+			`,
+			camelJSON: `
+			{
+				"Kind": "service-resolver",
+				"Name": "main"
+			}
+			`,
 			expect: &api.ServiceResolverConfigEntry{
 				Kind: "service-resolver",
 				Name: "main",
@@ -626,7 +1040,7 @@ func TestParseConfigEntry(t *testing.T) {
 	} {
 		tc := tc
 
-		testbody := func(t *testing.T, body string) {
+		testbody := func(t *testing.T, body string, expect api.ConfigEntry) {
 			t.Helper()
 			got, err := parseConfigEntry(body)
 			if tc.expectErr != "" {
@@ -635,16 +1049,34 @@ func TestParseConfigEntry(t *testing.T) {
 				requireContainsLower(t, err.Error(), tc.expectErr)
 			} else {
 				require.NoError(t, err)
-				require.Equal(t, tc.expect, got)
+				require.Equal(t, expect, got)
 			}
 		}
 
-		t.Run(tc.name+" (snake case)", func(t *testing.T) {
+		t.Run(tc.name+" (hcl snake case)", func(t *testing.T) {
-			testbody(t, tc.snake)
+			testbody(t, tc.snake, tc.expect)
 		})
-		t.Run(tc.name+" (camel case)", func(t *testing.T) {
+		t.Run(tc.name+" (hcl camel case)", func(t *testing.T) {
-			testbody(t, tc.camel)
+			testbody(t, tc.camel, tc.expect)
 		})
+		if tc.snakeJSON != "" {
+			t.Run(tc.name+" (json snake case)", func(t *testing.T) {
+				if tc.expectJSON != nil {
+					testbody(t, tc.snakeJSON, tc.expectJSON)
+				} else {
+					testbody(t, tc.snakeJSON, tc.expect)
+				}
+			})
+		}
+		if tc.camelJSON != "" {
+			t.Run(tc.name+" (json camel case)", func(t *testing.T) {
+				if tc.expectJSON != nil {
+					testbody(t, tc.camelJSON, tc.expectJSON)
+				} else {
+					testbody(t, tc.camelJSON, tc.expect)
+				}
+			})
+		}
 	}
 }
 

command/config/write/decode_shim.go

@@ -0,0 +1,108 @@
+package write
+
+import (
+	"encoding/json"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/hashicorp/hcl"
+)
+
+// hclDecode is a modified version of hcl.Decode just for the super general
+// purposes here. There's some strange bug in how hcl.Decode decodes json where
+//
+// { "sub" : { "v1" : { "field" : "value1" }, "v2" : { "field" : "value2" } } }
+//
+// hcl.Decode-s into:
+//
+// map[string]interface {}{
+// 	"sub":[]map[string]interface {}{
+// 		map[string]interface {}{
+// 			"v1":[]map[string]interface {}{
+// 				map[string]interface {}{
+// 					"field":"value1"
+// 				}
+// 			}
+// 		},
+// 		map[string]interface {}{
+// 			"v2":[]map[string]interface {}{
+// 				map[string]interface {}{
+// 					"field":"value2"
+// 				}
+// 			}
+// 		}
+// 	}
+// }
+//
+// but json.Unmarshal-s into the more expected:
+//
+// map[string]interface {}{
+// 	"sub":map[string]interface {}{
+// 		"v1":map[string]interface {}{
+// 			"field":"value1"
+// 		},
+// 		"v2":map[string]interface {}{
+// 			"field":"value2"
+// 		}
+// 	}
+// }
+//
+// The strange part is that the following HCL:
+//
+// sub { "v1" = { field = "value1" }, "v2" = { field = "value2" } }
+//
+// hcl.Decode-s into:
+//
+// map[string]interface {}{
+// 	"sub":[]map[string]interface {}{
+// 		map[string]interface {}{
+// 			"v1":[]map[string]interface {}{
+// 				map[string]interface {}{
+// 					"field":"value1"
+// 				}
+// 			},
+// 			"v2":[]map[string]interface {}{
+// 				map[string]interface {}{
+// 					"field":"value2"
+// 				}
+// 			}
+// 		}
+// 	}
+// }
+//
+// Which is the "correct" value assuming you did the patch-slice-of-maps correction.
+//
+// Given that HCLv1 is basically frozen and the HCL part of it is fine instead
+// of trying to track down a weird bug we'll bypass the weird JSON decoder and just use
+// the stdlib one.
+func hclDecode(out interface{}, in string) error {
+	data := []byte(in)
+	if isHCL(data) {
+		return hcl.Decode(out, in)
+	}
+
+	return json.Unmarshal(data, out)
+}
+
+// this is an inlined variant of hcl.lexMode()
+func isHCL(v []byte) bool {
+	var (
+		r      rune
+		w      int
+		offset int
+	)
+
+	for {
+		r, w = utf8.DecodeRune(v[offset:])
+		offset += w
+		if unicode.IsSpace(r) {
+			continue
+		}
+		if r == '{' {
+			return false
+		}
+		break
+	}
+
+	return true
+}