build: auto install correct version of protoc locally (#12651)

.circleci/config.yml

@@ -226,15 +226,6 @@ jobs:
       <<: *ENVIRONMENT
     steps:
       - checkout
-      - run:
-          name: Install protobuf
-          command: |
-            protoc_version="$(make print-PROTOC_VERSION)"
-            wget https://github.com/protocolbuffers/protobuf/releases/download/v${protoc_version}/protoc-${protoc_version}-linux-x86_64.zip
-            sudo unzip -d /usr/local protoc-*.zip
-            sudo chmod +x /usr/local/bin/protoc
-            sudo chmod -R a+Xr /usr/local/include/google/
-            rm protoc-*.zip
       - run:
           name: Install protobuf
           command: make proto-tools

.gitignore

@@ -6,6 +6,7 @@
 *.test
 .envrc
 .gotools
+.protobuf
 .vagrant/
 /pkg
 bin/

GNUmakefile

@@ -12,6 +12,11 @@ GOTOOLS = \
 	github.com/hashicorp/lint-consul-retry@master
 
 PROTOC_VERSION=3.12.3
+PROTOC_OS := $(shell if test "$(uname)" == "Darwin"; then echo osx; else echo linux; fi)
+PROTOC_ZIP := protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-x86_64.zip
+PROTOC_URL := https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/$(PROTOC_ZIP)
+PROTOC_ROOT := .protobuf/protoc-$(PROTOC_OS)-$(PROTOC_VERSION)
+PROTOC_BIN := $(PROTOC_ROOT)/bin/protoc
 GOPROTOVERSION?=$(shell grep github.com/golang/protobuf go.mod | awk '{print $$2}')
 GOPROTOTOOLS = \
 	github.com/golang/protobuf/protoc-gen-go@$(GOPROTOVERSION) \
@@ -33,7 +38,7 @@ GIT_DIRTY?=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true
 GIT_IMPORT=github.com/hashicorp/consul/version
 GOLDFLAGS=-X $(GIT_IMPORT).GitCommit=$(GIT_COMMIT)$(GIT_DIRTY)
 
-PROTOFILES?=$(shell find . -name '*.proto' | grep -v 'vendor/')
+PROTOFILES?=$(shell find . -name '*.proto' | grep -v 'vendor/' | grep -v '.protobuf' )
 PROTOGOFILES=$(PROTOFILES:.proto=.pb.go)
 PROTOGOBINFILES=$(PROTOFILES:.proto=.pb.binary.go)
 
@@ -353,22 +358,25 @@ else
 	@go test -v ./agent -run Vault
 endif
 
-.PHONY: protoc-check
+.PHONY: protoc-install
-protoc-check:
+protoc-install:
-	$(info checking protocol buffer compiler version (expect: $(PROTOC_VERSION)))
+	$(info locally installing protocol buffer compiler version if needed (expect: $(PROTOC_VERSION)))
-	@if ! command -v protoc &>/dev/null; then \
+	@if [[ ! -x $(PROTOC_ROOT)/bin/protoc ]]; then \
-		echo "ERROR: protoc is not installed; please install version $(PROTOC_VERSION)" >&2 ; \
+		mkdir -p .protobuf/tmp ; \
-		exit 1 ; \
+		if [[ ! -f .protobuf/tmp/$(PROTOC_ZIP) ]]; then \
-	fi
+			( cd .protobuf/tmp && curl -sSL "$(PROTOC_URL)" -o "$(PROTOC_ZIP)" ) ; \
-	@if [[ "$$(protoc --version | cut -d' ' -f2)" != "$(PROTOC_VERSION)" ]]; then \
+		fi ; \
-		echo "ERROR: protoc version $(PROTOC_VERSION) is required" >&2 ; \
+		mkdir -p $(PROTOC_ROOT) ; \
+		unzip -d $(PROTOC_ROOT) .protobuf/tmp/$(PROTOC_ZIP) ; \
+		chmod -R a+Xr $(PROTOC_ROOT) ; \
+		chmod +x $(PROTOC_ROOT)/bin/protoc ; \
 	fi
 
-proto: protoc-check $(PROTOGOFILES) $(PROTOGOBINFILES)
+proto: protoc-install $(PROTOGOFILES) $(PROTOGOBINFILES)
 	@echo "Generated all protobuf Go files"
 
 %.pb.go %.pb.binary.go: %.proto
-	@$(SHELL) $(CURDIR)/build-support/scripts/proto-gen.sh --grpc "$<"
+	@$(SHELL) $(CURDIR)/build-support/scripts/proto-gen.sh --grpc --protoc-bin "$(PROTOC_BIN)" "$<"
 
 # utility to echo a makefile variable (i.e. 'make print-PROTOC_VERSION')
 print-%  : ; @echo $($*)

build-support/scripts/proto-gen.sh

@@ -23,6 +23,7 @@ Description:
    generated code.
 
 Options:
+   --protoc-bin             Path to protoc.
    --import-replace         Replace imports of google types with those from the protobuf repo.
    --grpc                   Enable the gRPC plugin
    -h | --help              Print this help text.
@@ -38,6 +39,7 @@ function err_usage {
 function main {
    local -i grpc=0
    local    proto_path=
+   local    protoc_bin=
 
    while test $# -gt 0
    do
@@ -50,6 +52,10 @@ function main {
             grpc=1
             shift
             ;;
+         --protoc-bin )
+            protoc_bin="$2"
+            shift 2
+            ;;
          * )
             proto_path="$1"
             shift
@@ -63,6 +69,17 @@ function main {
       return 1
    fi
 
+   if test -z "${protoc_bin}"
+   then
+       protoc_bin="$(command -v protoc)"
+       if test -z "${protoc_bin}"
+       then
+           err_usage "ERROR: no proto-bin specified and protoc could not be discovered"
+           return 1
+       fi
+   fi
+
+
    go mod download
 
    local golang_proto_path=$(go list -f '{{ .Dir }}' -m github.com/golang/protobuf)
@@ -92,14 +109,14 @@ function main {
    #  -I="${golang_proto_path}/protobuf" \
    local -i ret=0
    status_stage "Generating ${proto_path} into ${proto_go_path} and ${proto_go_bin_path} ${mog_input_path}/*.gen.go"
-   echo "debug_run protoc \
+   echo "debug_run ${protoc_bin} \
           -I=\"${golang_proto_path}\" \
           -I=\"${golang_proto_mod_path}\" \
           -I=\"${SOURCE_DIR}\" \
           --go_out=\"${go_proto_out}${SOURCE_DIR}\" \
           --go-binary_out=\"${SOURCE_DIR}\" \
           \"${proto_path}\""
-   debug_run protoc \
+   debug_run ${protoc_bin} \
       -I="${golang_proto_path}" \
       -I="${golang_proto_mod_path}" \
       -I="${SOURCE_DIR}" \