api: add option to set TLS options in-memory for API client (#7093)

This PR adds the option to set in-memory certificates to the API client instead of requiring the certificate to be stored on disk in a file.

This allows us to define API client TLS options per Consul secret backend in Vault.
Related issue hashicorp/vault#4800

api/api.go

@@ -327,14 +327,26 @@ type TLSConfig struct {
 	// Consul communication, defaults to the system bundle if not specified.
 	CAPath string
 
+	// CAPem is the optional PEM-encoded CA certificate used for Consul
+	// communication, defaults to the system bundle if not specified.
+	CAPem []byte
+
 	// CertFile is the optional path to the certificate for Consul
 	// communication. If this is set then you need to also set KeyFile.
 	CertFile string
 
+	// CertPEM is the optional PEM-encoded certificate for Consul
+	// communication. If this is set then you need to also set KeyPEM.
+	CertPEM []byte
+
 	// KeyFile is the optional path to the private key for Consul communication.
 	// If this is set then you need to also set CertFile.
 	KeyFile string
 
+	// KeyPEM is the optional PEM-encoded private key for Consul communication.
+	// If this is set then you need to also set CertPEM.
+	KeyPEM []byte
+
 	// InsecureSkipVerify if set to true will disable TLS host verification.
 	InsecureSkipVerify bool
 }
@@ -458,18 +470,31 @@ func SetupTLSConfig(tlsConfig *TLSConfig) (*tls.Config, error) {
 		tlsClientConfig.ServerName = server
 	}
 
+	if len(tlsConfig.CertPEM) != 0 && len(tlsConfig.KeyPEM) != 0 {
+		tlsCert, err := tls.X509KeyPair(tlsConfig.CertPEM, tlsConfig.KeyPEM)
+		if err != nil {
+			return nil, err
+		}
+		tlsClientConfig.Certificates = []tls.Certificate{tlsCert}
+	} else if len(tlsConfig.CertPEM) != 0 || len(tlsConfig.KeyPEM) != 0 {
+		return nil, fmt.Errorf("both client cert and client key must be provided")
+	}
+
 	if tlsConfig.CertFile != "" && tlsConfig.KeyFile != "" {
 		tlsCert, err := tls.LoadX509KeyPair(tlsConfig.CertFile, tlsConfig.KeyFile)
 		if err != nil {
 			return nil, err
 		}
 		tlsClientConfig.Certificates = []tls.Certificate{tlsCert}
+	} else if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" {
+		return nil, fmt.Errorf("both client cert and client key must be provided")
 	}
 
-	if tlsConfig.CAFile != "" || tlsConfig.CAPath != "" {
+	if tlsConfig.CAFile != "" || tlsConfig.CAPath != "" || len(tlsConfig.CAPem) != 0 {
 		rootConfig := &rootcerts.Config{
-			CAFile: tlsConfig.CAFile,
-			CAPath: tlsConfig.CAPath,
+			CAFile:        tlsConfig.CAFile,
+			CAPath:        tlsConfig.CAPath,
+			CACertificate: tlsConfig.CAPem,
 		}
 		if err := rootcerts.ConfigureTLS(tlsClientConfig, rootConfig); err != nil {
 			return nil, err

api/api_test.go

@@ -4,6 +4,7 @@ import (
 	crand "crypto/rand"
 	"crypto/tls"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -575,6 +576,36 @@ func TestAPI_SetupTLSConfig(t *testing.T) {
 	if len(cc.RootCAs.Subjects()) != 2 {
 		t.Fatalf("didn't load root CAs")
 	}
+
+	// Load certs in-memory
+	certPEM, err := ioutil.ReadFile("../test/hostname/Alice.crt")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	keyPEM, err := ioutil.ReadFile("../test/hostname/Alice.key")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	caPEM, err := ioutil.ReadFile("../test/hostname/CertAuth.crt")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Setup config with in-memory certs
+	cc, err = SetupTLSConfig(&TLSConfig{
+		CertPEM: certPEM,
+		KeyPEM:  keyPEM,
+		CAPem:   caPEM,
+	})
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if len(cc.Certificates) != 1 {
+		t.Fatalf("missing certificate: %v", cc.Certificates)
+	}
+	if cc.RootCAs == nil {
+		t.Fatalf("didn't load root CAs")
+	}
 }
 
 func TestAPI_ClientTLSOptions(t *testing.T) {

api/go.mod

@@ -7,7 +7,7 @@ replace github.com/hashicorp/consul/sdk => ../sdk
 require (
 	github.com/hashicorp/consul/sdk v0.2.0
 	github.com/hashicorp/go-cleanhttp v0.5.1
-	github.com/hashicorp/go-rootcerts v1.0.0
+	github.com/hashicorp/go-rootcerts v1.0.2
 	github.com/hashicorp/go-uuid v1.0.1
 	github.com/hashicorp/serf v0.8.2
 	github.com/mitchellh/mapstructure v1.1.2

api/go.sum

@@ -21,6 +21,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -43,6 +45,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=

go.sum

@@ -151,6 +151,8 @@ github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82k
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8=
 github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=

vendor/github.com/hashicorp/go-rootcerts/README.md

@@ -28,8 +28,9 @@ Here's a snippet demonstrating how this library is meant to be used:
 func httpClient() (*http.Client, error)
 	tlsConfig := &tls.Config{}
 	err := rootcerts.ConfigureTLS(tlsConfig, &rootcerts.Config{
-		CAFile: os.Getenv("MYAPP_CAFILE"),
-		CAPath: os.Getenv("MYAPP_CAPATH"),
+		CAFile:      os.Getenv("MYAPP_CAFILE"),
+		CAPath:      os.Getenv("MYAPP_CAPATH"),
+		Certificate: os.Getenv("MYAPP_CERTIFICATE"),
 	})
 	if err != nil {
 		return nil, err

vendor/github.com/hashicorp/go-rootcerts/rootcerts.go

@@ -3,21 +3,26 @@ package rootcerts
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 )
 
-// Config determines where LoadCACerts will load certificates from. When both
-// CAFile and CAPath are blank, this library's functions will either load
+// Config determines where LoadCACerts will load certificates from. When CAFile,
+// CACertificate and CAPath are blank, this library's functions will either load
 // system roots explicitly and return them, or set the CertPool to nil to allow
 // Go's standard library to load system certs.
 type Config struct {
 	// CAFile is a path to a PEM-encoded certificate file or bundle. Takes
-	// precedence over CAPath.
+	// precedence over CACertificate and CAPath.
 	CAFile string
 
+	// CACertificate is a PEM-encoded certificate or bundle. Takes precedence
+	// over CAPath.
+	CACertificate []byte
+
 	// CAPath is a path to a directory populated with PEM-encoded certificates.
 	CAPath string
 }
@@ -44,6 +49,9 @@ func LoadCACerts(c *Config) (*x509.CertPool, error) {
 	if c.CAFile != "" {
 		return LoadCAFile(c.CAFile)
 	}
+	if len(c.CACertificate) != 0 {
+		return AppendCertificate(c.CACertificate)
+	}
 	if c.CAPath != "" {
 		return LoadCAPath(c.CAPath)
 	}
@@ -68,6 +76,18 @@ func LoadCAFile(caFile string) (*x509.CertPool, error) {
 	return pool, nil
 }
 
+// AppendCertificate appends an in-memory PEM-encoded certificate or bundle and returns a pool.
+func AppendCertificate(ca []byte) (*x509.CertPool, error) {
+	pool := x509.NewCertPool()
+
+	ok := pool.AppendCertsFromPEM(ca)
+	if !ok {
+		return nil, errors.New("Error appending CA: Couldn't parse PEM")
+	}
+
+	return pool, nil
+}
+
 // LoadCAPath walks the provided path and loads all certificates encounted into
 // a pool.
 func LoadCAPath(caPath string) (*x509.CertPool, error) {

vendor/modules.txt

@@ -209,7 +209,7 @@ github.com/hashicorp/go-raftchunking
 github.com/hashicorp/go-raftchunking/types
 # github.com/hashicorp/go-retryablehttp v0.5.4
 github.com/hashicorp/go-retryablehttp
-# github.com/hashicorp/go-rootcerts v1.0.1
+# github.com/hashicorp/go-rootcerts v1.0.2
 github.com/hashicorp/go-rootcerts
 # github.com/hashicorp/go-sockaddr v1.0.2
 github.com/hashicorp/go-sockaddr/template