consul/agent/grpc-external/services/dataplane/get_envoy_bootstrap_params_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package dataplane

import (
	"context"
	"fmt"
	"testing"

	"github.com/hashicorp/go-hclog"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/types/known/structpb"

	svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing"
	"github.com/hashicorp/consul/internal/catalog"
	"github.com/hashicorp/consul/internal/mesh"
	"github.com/hashicorp/consul/internal/resource"
	"github.com/hashicorp/consul/internal/resource/resourcetest"
	pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1"
	pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1"
	"github.com/hashicorp/consul/proto-public/pbresource"
	"github.com/hashicorp/consul/proto/private/prototest"

	"github.com/hashicorp/consul/acl"
	"github.com/hashicorp/consul/acl/resolver"
	external "github.com/hashicorp/consul/agent/grpc-external"
	"github.com/hashicorp/consul/agent/grpc-external/testutils"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/proto-public/pbdataplane"
)

const (
	testToken        = "acl-token-get-envoy-bootstrap-params"
	testServiceName  = "web"
	proxyServiceID   = "web-proxy"
	nodeName         = "foo"
	nodeID           = "2980b72b-bd9d-9d7b-d4f9-951bf7508d95"
	proxyConfigKey   = "envoy_dogstatsd_url"
	proxyConfigValue = "udp://127.0.0.1:8125"
	serverDC         = "dc1"

	protocolKey       = "protocol"
	connectTimeoutKey = "local_connect_timeout_ms"
	requestTimeoutKey = "local_request_timeout_ms"

	proxyDefaultsProtocol         = "http"
	proxyDefaultsRequestTimeout   = 1111
	serviceDefaultsProtocol       = "tcp"
	serviceDefaultsConnectTimeout = 4444

	testAccessLogs = "{\"name\":\"Consul Listener Filter Log\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog\",\"logFormat\":{\"jsonFormat\":{\"custom_field\":\"%START_TIME%\"}}}}"
)

func testRegisterRequestProxy(t *testing.T) *structs.RegisterRequest {
	return &structs.RegisterRequest{
		Datacenter: serverDC,
		Node:       nodeName,
		ID:         nodeID,
		Address:    "127.0.0.1",
		Service: &structs.NodeService{
			Kind:    structs.ServiceKindConnectProxy,
			Service: proxyServiceID,
			ID:      proxyServiceID,
			Address: "127.0.0.2",
			Port:    2222,
			Proxy: structs.ConnectProxyConfig{
				DestinationServiceName: testServiceName,
				Config: map[string]interface{}{
					proxyConfigKey: proxyConfigValue,
				},
			},
			EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
		},
	}
}

func testRegisterIngressGateway(t *testing.T) *structs.RegisterRequest {
	registerReq := structs.TestRegisterIngressGateway(t)
	registerReq.ID = "2980b72b-bd9d-9d7b-d4f9-951bf7508d95"
	registerReq.Service.ID = registerReq.Service.Service
	registerReq.Service.Proxy.Config = map[string]interface{}{
		proxyConfigKey: proxyConfigValue,
	}
	return registerReq
}

func testProxyDefaults(t *testing.T, accesslogs bool) structs.ConfigEntry {
	pd := &structs.ProxyConfigEntry{
		Kind: structs.ProxyDefaults,
		Name: structs.ProxyConfigGlobal,
		Config: map[string]interface{}{
			protocolKey:       proxyDefaultsProtocol,
			requestTimeoutKey: proxyDefaultsRequestTimeout,
		},
	}
	if accesslogs {
		pd.AccessLogs.Enabled = true
		pd.AccessLogs.JSONFormat = "{ \"custom_field\": \"%START_TIME%\" }"
	}
	return pd
}

func testServiceDefaults(t *testing.T) structs.ConfigEntry {
	return &structs.ServiceConfigEntry{
		Kind:                  structs.ServiceDefaults,
		Name:                  testServiceName,
		Protocol:              serviceDefaultsProtocol,
		LocalConnectTimeoutMs: serviceDefaultsConnectTimeout,
	}
}

func requireConfigField(t *testing.T, resp *pbdataplane.GetEnvoyBootstrapParamsResponse, key string, value interface{}) {
	require.Contains(t, resp.Config.Fields, key)
	require.Equal(t, value, resp.Config.Fields[key])
}

func TestGetEnvoyBootstrapParams_Success(t *testing.T) {
	type testCase struct {
		name            string
		registerReq     *structs.RegisterRequest
		nodeID          bool
		proxyDefaults   structs.ConfigEntry
		serviceDefaults structs.ConfigEntry
	}

	run := func(t *testing.T, tc testCase) {
		store := testutils.TestStateStore(t, nil)
		idx := uint64(1)
		err := store.EnsureRegistration(idx, tc.registerReq)
		require.NoError(t, err)

		if tc.proxyDefaults != nil {
			idx++
			err := store.EnsureConfigEntry(idx, tc.proxyDefaults)
			require.NoError(t, err)
		}
		if tc.serviceDefaults != nil {
			idx++
			err := store.EnsureConfigEntry(idx, tc.serviceDefaults)
			require.NoError(t, err)
		}

		aclResolver := &MockACLResolver{}
		aclResolver.On("ResolveTokenAndDefaultMeta", testToken, mock.Anything, mock.Anything).
			Return(testutils.ACLServiceRead(t, tc.registerReq.Service.ID), nil)

		options := structs.QueryOptions{Token: testToken}
		ctx, err := external.ContextWithQueryOptions(context.Background(), options)
		require.NoError(t, err)

		server := NewServer(Config{
			GetStore:    func() StateStore { return store },
			Logger:      hclog.NewNullLogger(),
			ACLResolver: aclResolver,
			Datacenter:  serverDC,
		})
		client := testClient(t, server)

		req := &pbdataplane.GetEnvoyBootstrapParamsRequest{
			ServiceId: tc.registerReq.Service.ID,
			NodeSpec:  &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeName{NodeName: tc.registerReq.Node}}
		if tc.nodeID {
			req.NodeSpec = &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeId{NodeId: string(tc.registerReq.ID)}
		}
		resp, err := client.GetEnvoyBootstrapParams(ctx, req)
		require.NoError(t, err)

		if tc.registerReq.Service.IsGateway() {
			require.Equal(t, tc.registerReq.Service.Service, resp.Service)
		} else {
			require.Equal(t, tc.registerReq.Service.Proxy.DestinationServiceName, resp.Service)
		}

		require.Equal(t, serverDC, resp.Datacenter)
		require.Equal(t, tc.registerReq.EnterpriseMeta.PartitionOrDefault(), resp.Partition)
		require.Equal(t, tc.registerReq.EnterpriseMeta.NamespaceOrDefault(), resp.Namespace)
		requireConfigField(t, resp, proxyConfigKey, structpb.NewStringValue(proxyConfigValue))
		require.Equal(t, tc.registerReq.Node, resp.NodeName)

		if tc.serviceDefaults != nil && tc.proxyDefaults != nil {
			// service-defaults take precedence over proxy-defaults
			requireConfigField(t, resp, protocolKey, structpb.NewStringValue(serviceDefaultsProtocol))
			requireConfigField(t, resp, connectTimeoutKey, structpb.NewNumberValue(serviceDefaultsConnectTimeout))
			requireConfigField(t, resp, requestTimeoutKey, structpb.NewNumberValue(proxyDefaultsRequestTimeout))
		} else if tc.serviceDefaults != nil {
			requireConfigField(t, resp, protocolKey, structpb.NewStringValue(serviceDefaultsProtocol))
			requireConfigField(t, resp, connectTimeoutKey, structpb.NewNumberValue(serviceDefaultsConnectTimeout))
		} else if tc.proxyDefaults != nil {
			requireConfigField(t, resp, protocolKey, structpb.NewStringValue(proxyDefaultsProtocol))
			requireConfigField(t, resp, requestTimeoutKey, structpb.NewNumberValue(proxyDefaultsRequestTimeout))
		}

		if tc.proxyDefaults != nil {
			pd, ok := tc.proxyDefaults.(*structs.ProxyConfigEntry)
			require.True(t, ok, "Invalid Proxy Defaults")
			if pd.AccessLogs.Enabled {
				require.JSONEq(t, "{\"name\":\"Consul Listener Filter Log\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog\",\"logFormat\":{\"jsonFormat\":{\"custom_field\":\"%START_TIME%\"}}}}", resp.AccessLogs[0])
			}
		}

	}

	testCases := []testCase{
		{
			name:        "lookup service sidecar proxy by node name",
			registerReq: testRegisterRequestProxy(t),
		},
		{
			name:        "lookup service sidecar proxy by node ID",
			registerReq: testRegisterRequestProxy(t),
			nodeID:      true,
		},
		{
			name:        "lookup ingress gw service by node name",
			registerReq: testRegisterIngressGateway(t),
		},
		{
			name:        "lookup ingress gw service by node ID",
			registerReq: testRegisterIngressGateway(t),
			nodeID:      true,
		},
		{
			name:          "merge proxy defaults for sidecar proxy",
			registerReq:   testRegisterRequestProxy(t),
			proxyDefaults: testProxyDefaults(t, false),
		},
		{
			name:          "proxy defaults access logs",
			registerReq:   testRegisterRequestProxy(t),
			proxyDefaults: testProxyDefaults(t, true),
		},
		{
			name:            "merge service defaults for sidecar proxy",
			registerReq:     testRegisterRequestProxy(t),
			serviceDefaults: testServiceDefaults(t),
		},
		{
			name:            "merge proxy defaults and service defaults for sidecar proxy",
			registerReq:     testRegisterRequestProxy(t),
			serviceDefaults: testServiceDefaults(t),
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			run(t, tc)
		})
	}
}

func TestGetEnvoyBootstrapParams_Success_EnableV2(t *testing.T) {
	type testCase struct {
		name            string
		workloadData    *pbcatalog.Workload
		proxyCfgs       []*pbmesh.ProxyConfiguration
		expBootstrapCfg *pbmesh.BootstrapConfig
		expAccessLogs   string
	}

	run := func(t *testing.T, tc testCase) {
		resourceClient := svctest.RunResourceService(t, catalog.RegisterTypes, mesh.RegisterTypes)

		options := structs.QueryOptions{Token: testToken}
		ctx, err := external.ContextWithQueryOptions(context.Background(), options)
		require.NoError(t, err)

		aclResolver := &MockACLResolver{}

		server := NewServer(Config{
			Logger:            hclog.NewNullLogger(),
			ACLResolver:       aclResolver,
			Datacenter:        serverDC,
			EnableV2:          true,
			ResourceAPIClient: resourceClient,
		})
		client := testClient(t, server)

		// Add required fields to workload data.
		tc.workloadData.Addresses = []*pbcatalog.WorkloadAddress{
			{
				Host: "127.0.0.1",
			},
		}
		tc.workloadData.Ports = map[string]*pbcatalog.WorkloadPort{
			"tcp": {Port: 8080, Protocol: pbcatalog.Protocol_PROTOCOL_TCP},
		}
		workloadResource := resourcetest.Resource(catalog.WorkloadType, "test-workload").
			WithData(t, tc.workloadData).
			WithTenancy(resource.DefaultNamespacedTenancy()).
			Write(t, resourceClient)

		// Create any proxy cfg resources.
		for i, cfg := range tc.proxyCfgs {
			resourcetest.Resource(mesh.ProxyConfigurationType, fmt.Sprintf("proxy-cfg-%d", i)).
				WithData(t, cfg).
				WithTenancy(resource.DefaultNamespacedTenancy()).
				Write(t, resourceClient)
		}

		req := &pbdataplane.GetEnvoyBootstrapParamsRequest{
			ProxyId:   workloadResource.Id.Name,
			Namespace: workloadResource.Id.Tenancy.Namespace,
			Partition: workloadResource.Id.Tenancy.Partition,
		}

		aclResolver.On("ResolveTokenAndDefaultMeta", testToken, mock.Anything, mock.Anything).
			Return(testutils.ACLServiceRead(t, workloadResource.Id.Name), nil)

		resp, err := client.GetEnvoyBootstrapParams(ctx, req)
		require.NoError(t, err)

		require.Equal(t, tc.workloadData.Identity, resp.Identity)
		require.Equal(t, serverDC, resp.Datacenter)
		require.Equal(t, workloadResource.Id.Tenancy.Partition, resp.Partition)
		require.Equal(t, workloadResource.Id.Tenancy.Namespace, resp.Namespace)
		require.Equal(t, resp.NodeName, tc.workloadData.NodeName)
		prototest.AssertDeepEqual(t, tc.expBootstrapCfg, resp.BootstrapConfig)
		if tc.expAccessLogs != "" {
			require.JSONEq(t, tc.expAccessLogs, resp.AccessLogs[0])
		}
	}

	testCases := []testCase{
		{
			name: "workload without node",
			workloadData: &pbcatalog.Workload{
				Identity: "test-identity",
			},
			expBootstrapCfg: &pbmesh.BootstrapConfig{},
		},
		{
			name: "workload with node",
			workloadData: &pbcatalog.Workload{
				Identity: "test-identity",
				NodeName: "test-node",
			},
			expBootstrapCfg: &pbmesh.BootstrapConfig{},
		},
		{
			name: "single proxy configuration",
			workloadData: &pbcatalog.Workload{
				Identity: "test-identity",
			},
			proxyCfgs: []*pbmesh.ProxyConfiguration{
				{
					Workloads: &pbcatalog.WorkloadSelector{Names: []string{"test-workload"}},
					BootstrapConfig: &pbmesh.BootstrapConfig{
						DogstatsdUrl: "dogstats-url",
					},
				},
			},
			expBootstrapCfg: &pbmesh.BootstrapConfig{
				DogstatsdUrl: "dogstats-url",
			},
		},
		{
			name: "multiple proxy configurations",
			workloadData: &pbcatalog.Workload{
				Identity: "test-identity",
			},
			proxyCfgs: []*pbmesh.ProxyConfiguration{
				{
					Workloads: &pbcatalog.WorkloadSelector{Names: []string{"test-workload"}},
					BootstrapConfig: &pbmesh.BootstrapConfig{
						DogstatsdUrl: "dogstats-url",
					},
				},
				{
					Workloads: &pbcatalog.WorkloadSelector{Prefixes: []string{"test-"}},
					BootstrapConfig: &pbmesh.BootstrapConfig{
						StatsdUrl: "stats-url",
					},
					DynamicConfig: &pbmesh.DynamicConfig{
						AccessLogs: &pbmesh.AccessLogsConfig{
							Enabled:    true,
							JsonFormat: "{ \"custom_field\": \"%START_TIME%\" }",
						},
					},
				},

				{
					Workloads: &pbcatalog.WorkloadSelector{Names: []string{"not-test-workload"}},
					BootstrapConfig: &pbmesh.BootstrapConfig{
						PrometheusBindAddr: "prom-addr",
					},
				},
			},
			expBootstrapCfg: &pbmesh.BootstrapConfig{
				DogstatsdUrl: "dogstats-url",
				StatsdUrl:    "stats-url",
			},
			expAccessLogs: testAccessLogs,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			run(t, tc)
		})
	}
}

func TestGetEnvoyBootstrapParams_Error(t *testing.T) {
	type testCase struct {
		name            string
		req             *pbdataplane.GetEnvoyBootstrapParamsRequest
		expectedErrCode codes.Code
		expecteErrMsg   string
	}

	run := func(t *testing.T, tc testCase) {
		aclResolver := &MockACLResolver{}

		aclResolver.On("ResolveTokenAndDefaultMeta", testToken, mock.Anything, mock.Anything).
			Return(testutils.ACLServiceRead(t, proxyServiceID), nil)

		options := structs.QueryOptions{Token: testToken}
		ctx, err := external.ContextWithQueryOptions(context.Background(), options)
		require.NoError(t, err)

		store := testutils.TestStateStore(t, nil)
		registerReq := testRegisterRequestProxy(t)
		err = store.EnsureRegistration(1, registerReq)
		require.NoError(t, err)

		server := NewServer(Config{
			GetStore:    func() StateStore { return store },
			Logger:      hclog.NewNullLogger(),
			ACLResolver: aclResolver,
		})
		client := testClient(t, server)

		resp, err := client.GetEnvoyBootstrapParams(ctx, tc.req)
		require.Nil(t, resp)
		require.Error(t, err)
		errStatus, ok := status.FromError(err)
		require.True(t, ok)
		require.Equal(t, tc.expectedErrCode.String(), errStatus.Code().String())
		require.Equal(t, tc.expecteErrMsg, errStatus.Message())
	}

	testCases := []testCase{
		{
			name: "lookup-service-by-unregistered-node-name",
			req: &pbdataplane.GetEnvoyBootstrapParamsRequest{
				ServiceId: proxyServiceID,
				NodeSpec:  &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeName{NodeName: "blah"}},
			expectedErrCode: codes.NotFound,
			expecteErrMsg:   "node not found",
		},
		{
			name: "lookup-service-by-unregistered-node-id",
			req: &pbdataplane.GetEnvoyBootstrapParamsRequest{
				ServiceId: proxyServiceID,
				NodeSpec:  &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeId{NodeId: "5980b72b-bd9d-9d7b-d4f9-951bf7508d98"}},
			expectedErrCode: codes.NotFound,
			expecteErrMsg:   "node not found",
		},
		{
			name: "lookup-service-by-unregistered-service",
			req: &pbdataplane.GetEnvoyBootstrapParamsRequest{
				ServiceId: "blah-service",
				NodeSpec:  &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeName{NodeName: nodeName}},
			expectedErrCode: codes.NotFound,
			expecteErrMsg:   "Service not found",
		},
		{
			name: "lookup-service-without-node-details",
			req: &pbdataplane.GetEnvoyBootstrapParamsRequest{
				ServiceId: proxyServiceID},
			expectedErrCode: codes.InvalidArgument,
			expecteErrMsg:   "Node ID or name required to lookup the service",
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			run(t, tc)
		})
	}

}

func TestGetEnvoyBootstrapParams_Error_EnableV2(t *testing.T) {
	type testCase struct {
		name            string
		expectedErrCode codes.Code
		expecteErrMsg   string
		workload        *pbresource.Resource
	}

	run := func(t *testing.T, tc testCase) {
		resourceClient := svctest.RunResourceService(t, catalog.RegisterTypes, mesh.RegisterTypes)

		options := structs.QueryOptions{Token: testToken}
		ctx, err := external.ContextWithQueryOptions(context.Background(), options)
		require.NoError(t, err)

		aclResolver := &MockACLResolver{}
		aclResolver.On("ResolveTokenAndDefaultMeta", testToken, mock.Anything, mock.Anything).
			Return(testutils.ACLServiceRead(t, "doesn't matter"), nil)

		server := NewServer(Config{
			Logger:            hclog.NewNullLogger(),
			ACLResolver:       aclResolver,
			Datacenter:        serverDC,
			EnableV2:          true,
			ResourceAPIClient: resourceClient,
		})
		client := testClient(t, server)

		var req pbdataplane.GetEnvoyBootstrapParamsRequest
		// Write the workload resource.
		if tc.workload != nil {
			_, err = resourceClient.Write(context.Background(), &pbresource.WriteRequest{
				Resource: tc.workload,
			})
			require.NoError(t, err)

			req = pbdataplane.GetEnvoyBootstrapParamsRequest{
				ProxyId:   tc.workload.Id.Name,
				Namespace: tc.workload.Id.Tenancy.Namespace,
				Partition: tc.workload.Id.Tenancy.Partition,
			}
		} else {
			req = pbdataplane.GetEnvoyBootstrapParamsRequest{
				ProxyId:   "not-found",
				Namespace: "default",
				Partition: "default",
			}
		}

		resp, err := client.GetEnvoyBootstrapParams(ctx, &req)
		require.Nil(t, resp)
		require.Error(t, err)
		errStatus, ok := status.FromError(err)
		require.True(t, ok)
		require.Equal(t, tc.expectedErrCode.String(), errStatus.Code().String())
		require.Equal(t, tc.expecteErrMsg, errStatus.Message())
	}

	workload := resourcetest.Resource(catalog.WorkloadType, "test-workload").
		WithData(t, &pbcatalog.Workload{
			Addresses: []*pbcatalog.WorkloadAddress{
				{Host: "127.0.0.1"},
			},
			Ports: map[string]*pbcatalog.WorkloadPort{
				"tcp": {Port: 8080},
			},
		}).
		WithTenancy(resource.DefaultNamespacedTenancy()).
		Build()

	testCases := []testCase{
		{
			name:            "workload doesn't exist",
			expectedErrCode: codes.NotFound,
			expecteErrMsg:   "resource not found",
		},
		{
			name:            "workload without identity",
			expectedErrCode: codes.InvalidArgument,
			expecteErrMsg:   "workload \"test-workload\" doesn't have identity associated with it",
			workload:        workload,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			run(t, tc)
		})
	}

}

func TestGetEnvoyBootstrapParams_Unauthenticated(t *testing.T) {
	// Mock the ACL resolver to return ErrNotFound.
	aclResolver := &MockACLResolver{}
	aclResolver.On("ResolveTokenAndDefaultMeta", mock.Anything, mock.Anything, mock.Anything).
		Return(resolver.Result{}, acl.ErrNotFound)

	options := structs.QueryOptions{Token: testToken}
	ctx, err := external.ContextWithQueryOptions(context.Background(), options)
	require.NoError(t, err)
	store := testutils.TestStateStore(t, nil)

	server := NewServer(Config{
		GetStore:    func() StateStore { return store },
		Logger:      hclog.NewNullLogger(),
		ACLResolver: aclResolver,
	})
	client := testClient(t, server)
	resp, err := client.GetEnvoyBootstrapParams(ctx, &pbdataplane.GetEnvoyBootstrapParamsRequest{})
	require.Error(t, err)
	require.Equal(t, codes.Unauthenticated.String(), status.Code(err).String())
	require.Nil(t, resp)
}

func TestGetEnvoyBootstrapParams_PermissionDenied(t *testing.T) {
	// Mock the ACL resolver to return a deny all authorizer
	aclResolver := &MockACLResolver{}
	aclResolver.On("ResolveTokenAndDefaultMeta", testToken, mock.Anything, mock.Anything).
		Return(testutils.ACLNoPermissions(t), nil)

	options := structs.QueryOptions{Token: testToken}
	ctx, err := external.ContextWithQueryOptions(context.Background(), options)
	require.NoError(t, err)

	store := testutils.TestStateStore(t, nil)
	registerReq := structs.TestRegisterRequestProxy(t)
	proxyServiceID := "web-sidecar-proxy"
	registerReq.Service.ID = proxyServiceID
	err = store.EnsureRegistration(1, registerReq)
	require.NoError(t, err)

	server := NewServer(Config{
		GetStore:    func() StateStore { return store },
		Logger:      hclog.NewNullLogger(),
		ACLResolver: aclResolver,
	})
	client := testClient(t, server)
	req := &pbdataplane.GetEnvoyBootstrapParamsRequest{
		ServiceId: proxyServiceID,
		NodeSpec:  &pbdataplane.GetEnvoyBootstrapParamsRequest_NodeName{NodeName: registerReq.Node}}

	resp, err := client.GetEnvoyBootstrapParams(ctx, req)
	require.Error(t, err)
	require.Equal(t, codes.PermissionDenied.String(), status.Code(err).String())
	require.Nil(t, resp)
}