NET-5600/container-test-acl-enabled (#18887)

* feat: add container tests for resource http api with acl enabled

* refactor: clean up

command/resource/client/client.go

@@ -805,6 +805,7 @@ func NewHttpClient(transport *http.Transport, tlsConf api.TLSConfig) (*http.Clie
 }
 
 // request is used to help build up a request
+// defined a custom object that includes use-case specific config
 type request struct {
 	config *api.Config
 	method string

test/integration/consul-container/libs/cluster/agent.go

@@ -45,6 +45,7 @@ type Agent interface {
 	Exec(ctx context.Context, cmd []string) (string, error)
 	DataDir() string
 	GetGRPCConn() *grpc.ClientConn
+	GetAPIClientConfig() api.Config
 }
 
 // Config is a set of configurations required to create a Agent

test/integration/consul-container/libs/cluster/container.go

@@ -68,6 +68,8 @@ type consulContainerNode struct {
 	nextConnectPortOffset int
 
 	info AgentInfo
+
+	apiClientConfig api.Config
 }
 
 func (c *consulContainerNode) GetPod() testcontainers.Container {
@@ -330,6 +332,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		}
 
 		node.client = apiClient
+		node.apiClientConfig = *apiConfig
 		node.clientAddr = clientAddr
 		node.clientCACertFile = clientCACertFile
 	}
@@ -445,6 +448,10 @@ func (c *consulContainerNode) GetIP() string {
 	return c.ip
 }
 
+func (c *consulContainerNode) GetAPIClientConfig() api.Config {
+	return c.apiClientConfig
+}
+
 func (c *consulContainerNode) RegisterTermination(f func() error) {
 	c.terminateFuncs = append(c.terminateFuncs, f)
 }

test/integration/consul-container/test/resource/http_api/acl_enabled_test.go

@@ -0,0 +1,223 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package resource
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_ServerAgent_Endpoints(t *testing.T) {
+	t.Parallel()
+
+	numOfServers := 1
+	numOfClients := 0
+	cluster, resourceClient := SetupClusterAndClient(t, makeClusterConfig(numOfServers, numOfClients, true), true)
+
+	resource := Resource{
+		HttpClient: resourceClient,
+	}
+
+	defer Terminate(t, cluster)
+
+	testCases := []testCase{
+		{
+			description: "should write resource successfully when token is provided",
+			operations: []operation{
+				{
+					action:           applyResource,
+					expectedErrorMsg: "",
+					includeToken:     true,
+				},
+				{
+					action:           readResource,
+					expectedErrorMsg: "",
+					includeToken:     true,
+				},
+				{
+					action:           listResource,
+					expectedErrorMsg: "",
+					includeToken:     true,
+				},
+				{
+					action:           deleteResource,
+					expectedErrorMsg: "",
+					includeToken:     true,
+				},
+			},
+			config: []config{
+				{
+					gvk:          demoGVK,
+					resourceName: "korn",
+					queryOptions: defaultTenancyQueryOptions,
+					payload:      demoPayload,
+				},
+				{
+					gvk:          demoGVK,
+					resourceName: "korn",
+					queryOptions: defaultTenancyQueryOptions,
+				},
+				{
+					gvk:          demoGVK,
+					queryOptions: defaultTenancyQueryOptions,
+				},
+				{
+					gvk:          demoGVK,
+					resourceName: "korn",
+					queryOptions: defaultTenancyQueryOptions,
+				},
+			},
+		},
+		{
+			description: "should return permission denied",
+			operations: []operation{
+				{
+					action:           applyResource,
+					expectedErrorMsg: "Unexpected response code: 403 (rpc error: code = PermissionDenied desc = Permission denied",
+					includeToken:     false,
+				},
+				{
+					action:           applyResource,
+					expectedErrorMsg: "",
+					includeToken:     true,
+				},
+				{
+					action:           readResource,
+					expectedErrorMsg: "Unexpected response code: 403 (rpc error: code = PermissionDenied desc = Permission denied",
+					includeToken:     false,
+				},
+				{
+					action:           listResource,
+					expectedErrorMsg: "Unexpected response code: 403 (rpc error: code = PermissionDenied desc = Permission denied",
+					includeToken:     false,
+				},
+				{
+					action:           deleteResource,
+					expectedErrorMsg: "Unexpected response code: 403 (rpc error: code = PermissionDenied desc = Permission denied",
+					includeToken:     false,
+				},
+			},
+			config: []config{
+				{
+					gvk:          demoGVK,
+					resourceName: "prince",
+					queryOptions: defaultTenancyQueryOptions,
+					payload:      demoPayload,
+				},
+				{
+					gvk:          demoGVK,
+					resourceName: "deleteme",
+					queryOptions: defaultTenancyQueryOptions,
+					payload:      demoPayload,
+				},
+				{
+					gvk:          demoGVK,
+					resourceName: "keith",
+					queryOptions: defaultTenancyQueryOptions,
+				},
+				{
+					gvk:          demoGVK,
+					queryOptions: defaultTenancyQueryOptions,
+				},
+				{
+					gvk:          demoGVK,
+					resourceName: "deleteme",
+					queryOptions: defaultTenancyQueryOptions,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			for i, op := range tc.operations {
+				if op.includeToken {
+					tc.config[i].queryOptions.Token = cluster.TokenBootstrap
+				}
+
+				err := op.action(&resource, tc.config[i])
+				if len(op.expectedErrorMsg) > 0 {
+					require.Error(t, err)
+					require.Contains(t, err.Error(), op.expectedErrorMsg)
+				} else {
+					require.NoError(t, err)
+				}
+			}
+		})
+	}
+}
+
+func Test_ClientAgent(t *testing.T) {
+	t.Parallel()
+
+	numOfServers := 1
+	numOfClients := 1
+	cluster, resourceClient := SetupClusterAndClient(t, makeClusterConfig(numOfServers, numOfClients, true), false)
+
+	resource := Resource{
+		HttpClient: resourceClient,
+	}
+
+	defer Terminate(t, cluster)
+
+	testCases := []testCase{
+		{
+			description: "should write resource successfully when token is provided",
+			operations: []operation{
+				{
+					action:           applyResource,
+					expectedErrorMsg: "",
+					includeToken:     true,
+				},
+			},
+			config: []config{
+				{
+					gvk:          demoGVK,
+					resourceName: "test",
+					queryOptions: defaultTenancyQueryOptions,
+					payload:      demoPayload,
+				},
+			},
+		},
+		{
+			description: "should return unauthorized when token is bad",
+			operations: []operation{
+				{
+					action:           applyResource,
+					expectedErrorMsg: "Unexpected response code: 403 (rpc error: code = PermissionDenied desc = Permission denied",
+					includeToken:     false,
+				},
+			},
+			config: []config{
+				{
+					gvk:          demoGVK,
+					resourceName: "test2",
+					queryOptions: defaultTenancyQueryOptions,
+					payload:      demoPayload,
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			for i, op := range tc.operations {
+				if op.includeToken {
+					tc.config[i].queryOptions.Token = cluster.TokenBootstrap
+				}
+
+				err := op.action(&resource, tc.config[i])
+				if len(op.expectedErrorMsg) > 0 {
+					require.Error(t, err)
+					require.Contains(t, err.Error(), op.expectedErrorMsg)
+				} else {
+					require.NoError(t, err)
+				}
+			}
+		})
+	}
+}

test/integration/consul-container/test/resource/http_api/client/client.go

@@ -0,0 +1,312 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+)
+
+// QueryOptions are used to parameterize a query
+type QueryOptions struct {
+	// Namespace overrides the `default` namespace
+	// Note: Namespaces are available only in Consul Enterprise
+	Namespace string
+
+	// Partition overrides the `default` partition
+	// Note: Partitions are available only in Consul Enterprise
+	Partition string
+
+	// Providing a peer name in the query option
+	Peer string
+
+	// RequireConsistent forces the read to be fully consistent.
+	// This is more expensive but prevents ever performing a stale
+	// read.
+	RequireConsistent bool
+
+	// ctx is an optional context pass through to the underlying HTTP
+	// request layer. Use Context() and WithContext() to manage this.
+	ctx context.Context
+
+	// Token is used to provide a per-request ACL token
+	// which overrides the agent's default token.
+	Token string
+}
+
+// Client provides a client to the Consul API
+type HttpClient struct {
+	modifyLock sync.RWMutex
+	headers    http.Header
+
+	config api.Config
+}
+
+// Headers gets the current set of headers used for requests. This returns a
+// copy; to modify it call AddHeader or SetHeaders.
+func (c *HttpClient) Headers() http.Header {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+
+	if c.headers == nil {
+		return nil
+	}
+
+	ret := make(http.Header)
+	for k, v := range c.headers {
+		ret[k] = append(ret[k], v...)
+	}
+
+	return ret
+}
+
+// NewClient returns a new client
+func NewClient(config *api.Config) (*HttpClient, error) {
+	// bootstrap the config
+	defConfig := api.DefaultConfig()
+
+	if config.Address == "" {
+		config.Address = defConfig.Address
+	}
+
+	if config.Scheme == "" {
+		config.Scheme = defConfig.Scheme
+	}
+
+	if config.Transport == nil {
+		config.Transport = defConfig.Transport
+	}
+
+	if config.HttpClient == nil {
+		var err error
+		config.HttpClient, err = NewHttpClient(config.Transport)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &HttpClient{config: *config, headers: make(http.Header)}, nil
+}
+
+// NewHttpClient returns an http client configured with the given Transport and TLS
+// config.
+func NewHttpClient(transport *http.Transport) (*http.Client, error) {
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	return client, nil
+}
+
+// request is used to help build up a request
+type request struct {
+	config *api.Config
+	method string
+	url    *url.URL
+	params url.Values
+	body   io.Reader
+	header http.Header
+	Obj    interface{}
+	ctx    context.Context
+}
+
+// setQueryOptions is used to annotate the request with
+// additional query options
+func (r *request) SetQueryOptions(q *QueryOptions) {
+	if q == nil {
+		return
+	}
+	if q.Namespace != "" {
+		// For backwards-compatibility with existing tests,
+		// use the short-hand query param name "ns"
+		// rather than the alternative long-hand "namespace"
+		r.params.Set("ns", q.Namespace)
+	}
+	if q.Partition != "" {
+		// For backwards-compatibility with existing tests,
+		// use the long-hand query param name "partition"
+		// rather than the alternative short-hand "ap"
+		r.params.Set("partition", q.Partition)
+	}
+	if q.Peer != "" {
+		r.params.Set("peer", q.Peer)
+	}
+
+	if q.RequireConsistent {
+		r.params.Set("consistent", "")
+	}
+
+	if q.Token != "" {
+		r.header.Set("X-Consul-Token", q.Token)
+	}
+
+	r.ctx = q.ctx
+}
+
+// toHTTP converts the request to an HTTP request
+func (r *request) toHTTP() (*http.Request, error) {
+	// Encode the query parameters
+	r.url.RawQuery = r.params.Encode()
+
+	// Check if we should encode the body
+	if r.body == nil && r.Obj != nil {
+		b, err := encodeBody(r.Obj)
+		if err != nil {
+			return nil, err
+		}
+		r.body = b
+	}
+
+	// Create the HTTP request
+	req, err := http.NewRequest(r.method, r.url.RequestURI(), r.body)
+	if err != nil {
+		return nil, err
+	}
+
+	// validate that socket communications that do not use the host, detect
+	// slashes in the host name and replace it with local host.
+	// this is required since go started validating req.host in 1.20.6 and 1.19.11.
+	// prior to that they would strip out the slashes for you.  They removed that
+	// behavior and added more strict validation as part of a CVE.
+	// This issue is being tracked by the Go team:
+	// https://github.com/golang/go/issues/61431
+	// If there is a resolution in this issue, we will remove this code.
+	// In the time being, this is the accepted workaround.
+	if strings.HasPrefix(r.url.Host, "/") {
+		r.url.Host = "localhost"
+	}
+
+	req.URL.Host = r.url.Host
+	req.URL.Scheme = r.url.Scheme
+	req.Host = r.url.Host
+	req.Header = r.header
+
+	// Content-Type must always be set when a body is present
+	// See https://github.com/hashicorp/consul/issues/10011
+	if req.Body != nil && req.Header.Get("Content-Type") == "" {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	// Setup auth
+	if r.config.HttpAuth != nil {
+		req.SetBasicAuth(r.config.HttpAuth.Username, r.config.HttpAuth.Password)
+	}
+	if r.ctx != nil {
+		return req.WithContext(r.ctx), nil
+	}
+
+	return req, nil
+}
+
+// newRequest is used to create a new request
+func (c *HttpClient) NewRequest(method, path string) *request {
+	r := &request{
+		config: &c.config,
+		method: method,
+		url: &url.URL{
+			Scheme: c.config.Scheme,
+			Host:   c.config.Address,
+			Path:   c.config.PathPrefix + path,
+		},
+		params: make(map[string][]string),
+		header: c.Headers(),
+	}
+
+	if c.config.Namespace != "" {
+		r.params.Set("ns", c.config.Namespace)
+	}
+	if c.config.Partition != "" {
+		r.params.Set("partition", c.config.Partition)
+	}
+	if c.config.Token != "" {
+		r.header.Set("X-Consul-Token", r.config.Token)
+	}
+	return r
+}
+
+// doRequest runs a request with our client
+func (c *HttpClient) DoRequest(r *request) (time.Duration, *http.Response, error) {
+	req, err := r.toHTTP()
+	if err != nil {
+		return 0, nil, err
+	}
+	start := time.Now()
+	resp, err := c.config.HttpClient.Do(req)
+	diff := time.Since(start)
+	return diff, resp, err
+}
+
+// DecodeBody is used to JSON decode a body
+func DecodeBody(resp *http.Response, out interface{}) error {
+	dec := json.NewDecoder(resp.Body)
+	return dec.Decode(out)
+}
+
+// encodeBody is used to encode a request body
+func encodeBody(obj interface{}) (io.Reader, error) {
+	buf := bytes.NewBuffer(nil)
+	enc := json.NewEncoder(buf)
+	if err := enc.Encode(obj); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+// requireOK is used to wrap doRequest and check for a 200
+func RequireOK(resp *http.Response) error {
+	return RequireHttpCodes(resp, 200)
+}
+
+// requireHttpCodes checks for the "allowable" http codes for a response
+func RequireHttpCodes(resp *http.Response, httpCodes ...int) error {
+	// if there is an http code that we require, return w no error
+	for _, httpCode := range httpCodes {
+		if resp.StatusCode == httpCode {
+			return nil
+		}
+	}
+
+	// if we reached here, then none of the http codes in resp matched any that we expected
+	// so err out
+	return generateUnexpectedResponseCodeError(resp)
+}
+
+// closeResponseBody reads resp.Body until EOF, and then closes it. The read
+// is necessary to ensure that the http.Client's underlying RoundTripper is able
+// to re-use the TCP connection. See godoc on net/http.Client.Do.
+func CloseResponseBody(resp *http.Response) error {
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return resp.Body.Close()
+}
+
+type StatusError struct {
+	Code int
+	Body string
+}
+
+func (e StatusError) Error() string {
+	return fmt.Sprintf("Unexpected response code: %d (%s)", e.Code, e.Body)
+}
+
+// generateUnexpectedResponseCodeError consumes the rest of the body, closes
+// the body stream and generates an error indicating the status code was
+// unexpected.
+func generateUnexpectedResponseCodeError(resp *http.Response) error {
+	var buf bytes.Buffer
+	io.Copy(&buf, resp.Body)
+	CloseResponseBody(resp)
+
+	trimmed := strings.TrimSpace(buf.String())
+	return StatusError{Code: resp.StatusCode, Body: trimmed}
+}

test/integration/consul-container/test/resource/http_api/helper.go

@@ -0,0 +1,213 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package resource
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
+	client "github.com/hashicorp/consul/test/integration/consul-container/test/resource/http_api/client"
+)
+
+func makeClusterConfig(numOfServers int, numOfClients int, aclEnabled bool) *libtopology.ClusterConfig {
+	return &libtopology.ClusterConfig{
+		NumServers:  numOfServers,
+		NumClients:  numOfClients,
+		LogConsumer: &libtopology.TestLogConsumer{},
+		BuildOpts: &libcluster.BuildOptions{
+			Datacenter: "dc1",
+			ACLEnabled: aclEnabled,
+		},
+		ApplyDefaultProxySettings: false,
+	}
+}
+
+type Resource struct {
+	HttpClient *client.HttpClient
+}
+
+type GVK struct {
+	Group   string
+	Version string
+	Kind    string
+}
+
+var demoGVK = GVK{
+	Group:   "demo",
+	Version: "v2",
+	Kind:    "Artist",
+}
+
+var defaultTenancyQueryOptions = client.QueryOptions{
+	Namespace: "default",
+	Partition: "default",
+	Peer:      "local",
+}
+
+type WriteRequest struct {
+	Metadata map[string]string
+	Data     map[string]any
+}
+
+var demoPayload = WriteRequest{
+	Metadata: map[string]string{
+		"foo": "bar",
+	},
+	Data: map[string]any{
+		"name": "cool",
+	},
+}
+
+type config struct {
+	gvk          GVK
+	resourceName string
+	queryOptions client.QueryOptions
+	payload      WriteRequest
+}
+
+type operation struct {
+	action           func(client *Resource, config config) error
+	expectedErrorMsg string
+	includeToken     bool
+}
+
+type testCase struct {
+	description string
+	operations  []operation
+	config      []config
+}
+
+var applyResource = func(resource *Resource, config config) error {
+	_, err := resource.Apply(&config.gvk, config.resourceName, &config.queryOptions, &config.payload)
+	return err
+}
+var readResource = func(resource *Resource, config config) error {
+	_, err := resource.Read(&config.gvk, config.resourceName, &config.queryOptions)
+	return err
+}
+var deleteResource = func(resource *Resource, config config) error {
+	err := resource.Delete(&config.gvk, config.resourceName, &config.queryOptions)
+	return err
+}
+var listResource = func(resource *Resource, config config) error {
+	_, err := resource.List(&config.gvk, &config.queryOptions)
+	return err
+}
+
+func (resource *Resource) Read(gvk *GVK, resourceName string, q *client.QueryOptions) (map[string]interface{}, error) {
+	r := resource.HttpClient.NewRequest("GET", strings.ToLower(fmt.Sprintf("/api/%s/%s/%s/%s", gvk.Group, gvk.Version, gvk.Kind, resourceName)))
+	r.SetQueryOptions(q)
+	_, resp, err := resource.HttpClient.DoRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer client.CloseResponseBody(resp)
+	if err := client.RequireOK(resp); err != nil {
+		return nil, err
+	}
+
+	var out map[string]interface{}
+	if err := client.DecodeBody(resp, &out); err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+func (resource *Resource) Delete(gvk *GVK, resourceName string, q *client.QueryOptions) error {
+	r := resource.HttpClient.NewRequest("DELETE", strings.ToLower(fmt.Sprintf("/api/%s/%s/%s/%s", gvk.Group, gvk.Version, gvk.Kind, resourceName)))
+	r.SetQueryOptions(q)
+	_, resp, err := resource.HttpClient.DoRequest(r)
+	if err != nil {
+		return err
+	}
+	defer client.CloseResponseBody(resp)
+	if err := client.RequireHttpCodes(resp, http.StatusNoContent); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (resource *Resource) Apply(gvk *GVK, resourceName string, q *client.QueryOptions, payload *WriteRequest) (*map[string]interface{}, error) {
+	url := strings.ToLower(fmt.Sprintf("/api/%s/%s/%s/%s", gvk.Group, gvk.Version, gvk.Kind, resourceName))
+
+	r := resource.HttpClient.NewRequest("PUT", url)
+	r.SetQueryOptions(q)
+	r.Obj = payload
+	_, resp, err := resource.HttpClient.DoRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer client.CloseResponseBody(resp)
+	if err := client.RequireOK(resp); err != nil {
+		return nil, err
+	}
+
+	var out map[string]interface{}
+
+	if err := client.DecodeBody(resp, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}
+
+type ListResponse struct {
+	Resources []map[string]interface{} `json:"resources"`
+}
+
+func (resource *Resource) List(gvk *GVK, q *client.QueryOptions) (*ListResponse, error) {
+	r := resource.HttpClient.NewRequest("GET", strings.ToLower(fmt.Sprintf("/api/%s/%s/%s", gvk.Group, gvk.Version, gvk.Kind)))
+	r.SetQueryOptions(q)
+	_, resp, err := resource.HttpClient.DoRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer client.CloseResponseBody(resp)
+	if err := client.RequireOK(resp); err != nil {
+		return nil, err
+	}
+
+	var out *ListResponse
+	if err := client.DecodeBody(resp, &out); err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+func SetupClusterAndClient(t *testing.T, clusterConfig *libtopology.ClusterConfig, getServerHttpClient bool) (*libcluster.Cluster, *client.HttpClient) {
+	cluster, _, _ := libtopology.NewCluster(t, clusterConfig)
+
+	// create a http api client for resource service
+	var resourceHttpClient *client.HttpClient
+	if getServerHttpClient {
+		apiClientConfig := cluster.Servers()[0].GetAPIClientConfig()
+		apiClientConfig.Token = ""
+		resourceClient, err := client.NewClient(&apiClientConfig)
+		require.NoError(t, err)
+
+		resourceHttpClient = resourceClient
+	} else {
+		apiClientConfig := cluster.Clients()[0].GetAPIClientConfig()
+		apiClientConfig.Token = ""
+		resourceClient, err := client.NewClient(&apiClientConfig)
+		require.NoError(t, err)
+
+		resourceHttpClient = resourceClient
+	}
+
+	return cluster, resourceHttpClient
+}
+
+func Terminate(t *testing.T, cluster *libcluster.Cluster) {
+	err := cluster.Terminate()
+	require.NoError(t, err)
+}