ci: add 'go test -race' job

Running every test with the race detector would add significant time to
CI. That additionaltime won't provide much value as many of the integration tests use
much of the same code.

For now we can run -race on some of the smaller packages. As we move
more code into smaller packages we should be able to add more packages
to the list that runs with '-race'.

For now this is running without parallelism, but we can enable that as
well when we need it.

boltdb fails the 'checkptr' check, which is automatically enabled by
'-race', so I've disabled checkptr as well.

.circleci/config.yml

@@ -114,7 +114,8 @@ jobs:
             PACKAGE_NAMES=$(go list -tags "$GOTAGS" ./... | circleci tests split --split-by=timings --timings-type=classname)
             echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
             echo $PACKAGE_NAMES
-            gotestsum --format=short-verbose \
+            gotestsum \
+              --format=short-verbose \
               --jsonfile /tmp/jsonfile/go-test-${CIRCLE_NODE_INDEX}.log \
               --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
               -tags="$GOTAGS" -p 2 \
@@ -134,6 +135,40 @@ jobs:
           # without it.
           command: bash <(curl -s https://codecov.io/bash) -C "$CIRCLE_SHA1"
 
+  go-test-race:
+    docker:
+      - image: *GOLANG_IMAGE
+    environment:
+      <<: *ENVIRONMENT
+      GOTAGS: "" # No tags for OSS but there are for enterprise
+      # GOMAXPROCS defaults to number of cores on underlying hardware, set
+      # explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
+      GOMAXPROCS: 4
+      # The medium resource class (default) boxes are 2 vCPUs, 4GB RAM
+      # https://circleci.com/docs/2.0/configuration-reference/#docker-executor
+      # but we can run a little over that limit.
+    steps:
+      - checkout
+      - run: *install-gotestsum
+      - run: go mod download
+      - run:
+          name: go test -race
+          command: |
+            mkdir -p $TEST_RESULTS_DIR
+            gotestsum \
+              --format=short-verbose \
+              --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- \
+              -tags="$GOTAGS" -p 2 \
+              -race -gcflags=all=-d=checkptr=0 \
+              ./agent/{ae,cache,checks,config,pool,proxycfg,router} \
+              ./agent/consul/{authmethod/...,autopilot,fsm,state,stream} \
+              ./snapshot
+
+      - store_test_results:
+          path: *TEST_RESULTS_DIR
+      - store_artifacts:
+          path: *TEST_RESULTS_DIR
+
   # split off a job for the API package since it is separate
   go-test-api:
     docker:
@@ -660,6 +695,7 @@ workflows:
           requires: [dev-build]
       - go-test-api:
           requires: [dev-build]
+      - go-test-race: *filter-ignore-non-go-branches
       - go-test-sdk: *filter-ignore-non-go-branches
 
   build-distros: