mirror of https://github.com/status-im/consul.git
Add cross-partition integration test
This commit is contained in:
freddygv
parent
4c9c1b52ce
commit
cc19f09f92
|
|
@ -2,4 +2,5 @@
|
|||
|
||||
export DEFAULT_REQUIRED_SERVICES="s1 s1-sidecar-proxy s2 s2-sidecar-proxy"
|
||||
export REQUIRED_SERVICES="${DEFAULT_REQUIRED_SERVICES}"
|
||||
export REQUIRE_SECONDARY=0
|
||||
export REQUIRE_SECONDARY=0
|
||||
export REQUIRE_PARTITIONS=0
|
||||
|
|
@ -115,14 +115,19 @@ function assert_proxy_presents_cert_uri {
|
|||
local SERVICENAME=$2
|
||||
local DC=${3:-primary}
|
||||
local NS=${4:-default}
|
||||
local PARTITION=${5:default}
|
||||
|
||||
CERT=$(retry_default get_cert $HOSTPORT)
|
||||
|
||||
echo "WANT SERVICE: ${NS}/${SERVICENAME}"
|
||||
echo "WANT SERVICE: ${PARTITION}/${NS}/${SERVICENAME}"
|
||||
echo "GOT CERT:"
|
||||
echo "$CERT"
|
||||
|
||||
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
|
||||
if [[ -z $PARTITION ]] || [[ $PARTITION = "default" ]]; then
|
||||
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
|
||||
else
|
||||
echo "$CERT" | grep -Eo "URI:spiffe://([a-zA-Z0-9-]+).consul/ap/${PARTITION}/ns/${NS}/dc/${DC}/svc/$SERVICENAME"
|
||||
fi
|
||||
}
|
||||
|
||||
function assert_dnssan_in_cert {
|
||||
|
|
|
|||
|
|
@ -40,39 +40,39 @@ function network_snippet {
|
|||
}
|
||||
|
||||
function init_workdir {
|
||||
local DC="$1"
|
||||
local CLUSTER="$1"
|
||||
|
||||
if test -z "$DC"
|
||||
if test -z "$CLUSTER"
|
||||
then
|
||||
DC=primary
|
||||
CLUSTER=primary
|
||||
fi
|
||||
|
||||
# Note, we use explicit set of dirs so we don't delete .gitignore. Also,
|
||||
# don't wipe logs between runs as they are already split and we need them to
|
||||
# upload as artifacts later.
|
||||
rm -rf workdir/${DC}
|
||||
mkdir -p workdir/${DC}/{consul,register,envoy,bats,statsd,data}
|
||||
rm -rf workdir/${CLUSTER}
|
||||
mkdir -p workdir/${CLUSTER}/{consul,register,envoy,bats,statsd,data}
|
||||
|
||||
# Reload consul config from defaults
|
||||
cp consul-base-cfg/*.hcl workdir/${DC}/consul/
|
||||
cp consul-base-cfg/*.hcl workdir/${CLUSTER}/consul/
|
||||
|
||||
# Add any overrides if there are any (no op if not)
|
||||
find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${DC}/consul \;
|
||||
find ${CASE_DIR} -maxdepth 1 -name '*.hcl' -type f -exec cp -f {} workdir/${CLUSTER}/consul \;
|
||||
|
||||
# Copy all the test files
|
||||
find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${DC}/bats \;
|
||||
# Copy DC specific bats
|
||||
cp helpers.bash workdir/${DC}/bats
|
||||
find ${CASE_DIR} -maxdepth 1 -name '*.bats' -type f -exec cp -f {} workdir/${CLUSTER}/bats \;
|
||||
# Copy CLUSTER specific bats
|
||||
cp helpers.bash workdir/${CLUSTER}/bats
|
||||
|
||||
# Add any DC overrides
|
||||
if test -d "${CASE_DIR}/${DC}"
|
||||
# Add any CLUSTER overrides
|
||||
if test -d "${CASE_DIR}/${CLUSTER}"
|
||||
then
|
||||
find ${CASE_DIR}/${DC} -type f -name '*.hcl' -exec cp -f {} workdir/${DC}/consul \;
|
||||
find ${CASE_DIR}/${DC} -type f -name '*.bats' -exec cp -f {} workdir/${DC}/bats \;
|
||||
find ${CASE_DIR}/${CLUSTER} -type f -name '*.hcl' -exec cp -f {} workdir/${CLUSTER}/consul \;
|
||||
find ${CASE_DIR}/${CLUSTER} -type f -name '*.bats' -exec cp -f {} workdir/${CLUSTER}/bats \;
|
||||
fi
|
||||
|
||||
# move all of the registration files OUT of the consul config dir now
|
||||
find workdir/${DC}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${DC}/register \;
|
||||
find workdir/${CLUSTER}/consul -type f -name 'service_*.hcl' -exec mv -f {} workdir/${CLUSTER}/register \;
|
||||
|
||||
# copy the ca-certs for SDS so we can verify the right ones are served
|
||||
mkdir -p workdir/test-sds-server/certs
|
||||
|
|
@ -80,7 +80,7 @@ function init_workdir {
|
|||
|
||||
if test -d "${CASE_DIR}/data"
|
||||
then
|
||||
cp -r ${CASE_DIR}/data/* workdir/${DC}/data
|
||||
cp -r ${CASE_DIR}/data/* workdir/${CLUSTER}/data
|
||||
fi
|
||||
|
||||
return 0
|
||||
|
|
@ -157,13 +157,48 @@ function start_consul {
|
|||
-client "0.0.0.0" >/dev/null
|
||||
}
|
||||
|
||||
function start_partitioned_client {
|
||||
local PARTITION=${1:-ap1}
|
||||
|
||||
# Start consul now as setup script needs it up
|
||||
docker_kill_rm consul-${PARTITION}
|
||||
|
||||
license="${CONSUL_LICENSE:-}"
|
||||
# load the consul license so we can pass it into the consul
|
||||
# containers as an env var in the case that this is a consul
|
||||
# enterprise test
|
||||
if test -z "$license" -a -n "${CONSUL_LICENSE_PATH:-}"
|
||||
then
|
||||
license=$(cat $CONSUL_LICENSE_PATH)
|
||||
fi
|
||||
|
||||
sh -c "rm -rf /workdir/${PARTITION}/data"
|
||||
|
||||
# Run consul and expose some ports to the host to make debugging locally a
|
||||
# bit easier.
|
||||
#
|
||||
docker run -d --name envoy_consul-${PARTITION}_1 \
|
||||
--net=envoy-tests \
|
||||
$WORKDIR_SNIPPET \
|
||||
--hostname "consul-${PARTITION}" \
|
||||
--network-alias "consul-${PARTITION}" \
|
||||
-e "CONSUL_LICENSE=$license" \
|
||||
consul-dev agent \
|
||||
-datacenter "primary" \
|
||||
-retry-join "consul-primary" \
|
||||
-grpc-port 8502 \
|
||||
-data-dir "/tmp/consul" \
|
||||
-config-dir "/workdir/${PARTITION}/consul" \
|
||||
-client "0.0.0.0" >/dev/null
|
||||
}
|
||||
|
||||
function pre_service_setup {
|
||||
local DC=${1:-primary}
|
||||
local CLUSTER=${1:-primary}
|
||||
|
||||
# Run test case setup (e.g. generating Envoy bootstrap, starting containers)
|
||||
if [ -f "${CASE_DIR}/${DC}/setup.sh" ]
|
||||
if [ -f "${CASE_DIR}/${CLUSTER}/setup.sh" ]
|
||||
then
|
||||
source ${CASE_DIR}/${DC}/setup.sh
|
||||
source ${CASE_DIR}/${CLUSTER}/setup.sh
|
||||
else
|
||||
source ${CASE_DIR}/setup.sh
|
||||
fi
|
||||
|
|
@ -184,29 +219,29 @@ function start_services {
|
|||
}
|
||||
|
||||
function verify {
|
||||
local DC=$1
|
||||
if test -z "$DC"; then
|
||||
DC=primary
|
||||
local CLUSTER="$1"
|
||||
if test -z "$CLUSTER"; then
|
||||
CLUSTER="primary"
|
||||
fi
|
||||
|
||||
# Execute tests
|
||||
res=0
|
||||
|
||||
# Nuke any previous case's verify container.
|
||||
docker_kill_rm verify-${DC}
|
||||
docker_kill_rm verify-${CLUSTER}
|
||||
|
||||
echo "Running ${DC} verification step for ${CASE_DIR}..."
|
||||
echo "Running ${CLUSTER} verification step for ${CASE_DIR}..."
|
||||
|
||||
# need to tell the PID 1 inside of the container that it won't be actual PID
|
||||
# 1 because we're using --pid=host so we use TINI_SUBREAPER
|
||||
if docker run --name envoy_verify-${DC}_1 -t \
|
||||
if docker run --name envoy_verify-${CLUSTER}_1 -t \
|
||||
-e TINI_SUBREAPER=1 \
|
||||
-e ENVOY_VERSION \
|
||||
$WORKDIR_SNIPPET \
|
||||
--pid=host \
|
||||
$(network_snippet $DC) \
|
||||
$(network_snippet $CLUSTER) \
|
||||
bats-verify \
|
||||
--pretty /workdir/${DC}/bats ; then
|
||||
--pretty /workdir/${CLUSTER}/bats ; then
|
||||
echogreen "✓ PASS"
|
||||
else
|
||||
echored "⨯ FAIL"
|
||||
|
|
@ -228,6 +263,11 @@ function capture_logs {
|
|||
then
|
||||
services="$services consul-secondary"
|
||||
fi
|
||||
if is_set $REQUIRE_PARTITIONS
|
||||
then
|
||||
services="$services consul-ap1"
|
||||
fi
|
||||
|
||||
|
||||
if [ -f "${CASE_DIR}/capture.sh" ]
|
||||
then
|
||||
|
|
@ -247,7 +287,7 @@ function stop_services {
|
|||
# Teardown
|
||||
docker_kill_rm $REQUIRED_SERVICES
|
||||
|
||||
docker_kill_rm consul-primary consul-secondary
|
||||
docker_kill_rm consul-primary consul-secondary consul-ap1
|
||||
}
|
||||
|
||||
function init_vars {
|
||||
|
|
@ -286,6 +326,10 @@ function run_tests {
|
|||
then
|
||||
init_workdir secondary
|
||||
fi
|
||||
if is_set $REQUIRE_PARTITIONS
|
||||
then
|
||||
init_workdir ap1
|
||||
fi
|
||||
|
||||
global_setup
|
||||
|
||||
|
|
@ -307,6 +351,10 @@ function run_tests {
|
|||
if is_set $REQUIRE_SECONDARY; then
|
||||
start_consul secondary
|
||||
fi
|
||||
if is_set $REQUIRE_PARTITIONS; then
|
||||
docker_consul "primary" admin-partition create -name ap1 > /dev/null
|
||||
start_partitioned_client ap1
|
||||
fi
|
||||
|
||||
echo "Setting up the primary datacenter"
|
||||
pre_service_setup primary
|
||||
|
|
@ -315,14 +363,20 @@ function run_tests {
|
|||
echo "Setting up the secondary datacenter"
|
||||
pre_service_setup secondary
|
||||
fi
|
||||
if is_set $REQUIRE_PARTITIONS; then
|
||||
echo "Setting up the non-default partition"
|
||||
pre_service_setup ap1
|
||||
fi
|
||||
|
||||
echo "Starting services"
|
||||
start_services
|
||||
|
||||
# Run the verify container and report on the output
|
||||
echo "Verifying the primary datacenter"
|
||||
verify primary
|
||||
|
||||
if is_set $REQUIRE_SECONDARY; then
|
||||
echo "Verifying the secondary datacenter"
|
||||
verify secondary
|
||||
fi
|
||||
}
|
||||
|
|
@ -378,7 +432,7 @@ function suite_teardown {
|
|||
docker_kill_rm $(grep "^function run_container_" $self_name | \
|
||||
sed 's/^function run_container_\(.*\) {/\1/g')
|
||||
|
||||
docker_kill_rm consul-primary consul-secondary
|
||||
docker_kill_rm consul-primary consul-secondary consul-ap1
|
||||
|
||||
if docker network inspect envoy-tests &>/dev/null ; then
|
||||
echo -n "Deleting network 'envoy-tests'..."
|
||||
|
|
@ -402,13 +456,13 @@ function run_container {
|
|||
|
||||
function common_run_container_service {
|
||||
local service="$1"
|
||||
local DC="$2"
|
||||
local CLUSTER="$2"
|
||||
local httpPort="$3"
|
||||
local grpcPort="$4"
|
||||
|
||||
docker run -d --name $(container_name_prev) \
|
||||
-e "FORTIO_NAME=${service}" \
|
||||
$(network_snippet $DC) \
|
||||
$(network_snippet $CLUSTER) \
|
||||
"${HASHICORP_DOCKER_PROXY}/fortio/fortio" \
|
||||
server \
|
||||
-http-port ":$httpPort" \
|
||||
|
|
@ -420,6 +474,10 @@ function run_container_s1 {
|
|||
common_run_container_service s1 primary 8080 8079
|
||||
}
|
||||
|
||||
function run_container_s1-ap1 {
|
||||
common_run_container_service s1 ap1 8080 8079
|
||||
}
|
||||
|
||||
function run_container_s2 {
|
||||
common_run_container_service s2 primary 8181 8179
|
||||
}
|
||||
|
|
@ -457,7 +515,7 @@ function run_container_s2-secondary {
|
|||
|
||||
function common_run_container_sidecar_proxy {
|
||||
local service="$1"
|
||||
local DC="$2"
|
||||
local CLUSTER="$2"
|
||||
|
||||
# Hot restart breaks since both envoys seem to interact with each other
|
||||
# despite separate containers that don't share IPC namespace. Not quite
|
||||
|
|
@ -465,10 +523,10 @@ function common_run_container_sidecar_proxy {
|
|||
# location?
|
||||
docker run -d --name $(container_name_prev) \
|
||||
$WORKDIR_SNIPPET \
|
||||
$(network_snippet $DC) \
|
||||
$(network_snippet $CLUSTER) \
|
||||
"${HASHICORP_DOCKER_PROXY}/envoyproxy/envoy:v${ENVOY_VERSION}" \
|
||||
envoy \
|
||||
-c /workdir/${DC}/envoy/${service}-bootstrap.json \
|
||||
-c /workdir/${CLUSTER}/envoy/${service}-bootstrap.json \
|
||||
-l debug \
|
||||
--disable-hot-restart \
|
||||
--drain-time-s 1 >/dev/null
|
||||
|
|
@ -477,6 +535,11 @@ function common_run_container_sidecar_proxy {
|
|||
function run_container_s1-sidecar-proxy {
|
||||
common_run_container_sidecar_proxy s1 primary
|
||||
}
|
||||
|
||||
function run_container_s1-ap1-sidecar-proxy {
|
||||
common_run_container_sidecar_proxy s1 ap1
|
||||
}
|
||||
|
||||
function run_container_s1-sidecar-proxy-consul-exec {
|
||||
docker run -d --name $(container_name) \
|
||||
$(network_snippet primary) \
|
||||
|
|
|
|||
Loading…
Reference in New Issue