allow automatically discovering aws ec2 consul servers

command/agent/command.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"io"
@@ -15,6 +16,10 @@ import (
 
 	"github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/datadog"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/watch"
 	"github.com/hashicorp/go-checkpoint"
@@ -94,6 +99,12 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.BoolVar(&cmdConfig.AtlasJoin, "atlas-join", false, "auto-join with Atlas")
 	cmdFlags.StringVar(&cmdConfig.AtlasEndpoint, "atlas-endpoint", "", "endpoint for Atlas integration")
 
+	cmdFlags.StringVar(&cmdConfig.AwsAccessKey, "aws-access-key", "", "AWS key for EC2 discovery")
+	cmdFlags.StringVar(&cmdConfig.AwsSecretKey, "aws-secret-key", "", "AWS secret for EC2 discovery")
+	cmdFlags.StringVar(&cmdConfig.AwsRegion, "aws-region", "", "Region to search for instances in")
+	cmdFlags.StringVar(&cmdConfig.EC2TagKey, "ec2-tag-key", "", "EC2 tag key to filter for server discovery")
+	cmdFlags.StringVar(&cmdConfig.EC2TagValue, "ec2-tag-value", "", "EC2 tag value to filter for server discovery")
+
 	cmdFlags.IntVar(&cmdConfig.Protocol, "protocol", -1, "protocol version")
 
 	cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
@@ -299,6 +310,32 @@ func (c *Command) readConfig() *Config {
 		c.Ui.Error("WARNING: Bootstrap mode enabled! Do not enable unless necessary")
 	}
 
+	if (config.AwsAccessKey != "" || config.AwsSecretKey != "") && (config.AwsAccessKey == "" && config.AwsSecretKey == "") {
+		c.Ui.Error("aws-acces-key and aws-secret-key are required together")
+		return nil
+	}
+
+	if config.EC2TagKey != "" || config.EC2TagValue != "" {
+		if config.EC2TagKey == "" && config.EC2TagValue == "" {
+			c.Ui.Error("ec2-tag-key and ec2-tag-value are required together")
+			return nil
+		}
+
+		if config.AwsRegion == "" {
+			c.Ui.Error("aws-region is required")
+			return nil
+		}
+
+		ec2servers, err := config.loadEc2Hosts()
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Unable to query EC2 insances: %s", err))
+			return nil
+		}
+		config.StartJoin = append(config.StartJoin, ec2servers...)
+		fmt.Println(config.StartJoin)
+		os.Exit(1)
+	}
+
 	// Set the version info
 	config.Revision = c.Revision
 	config.Version = c.Version
@@ -351,6 +388,46 @@ func (config *Config) verifyUniqueListeners() error {
 	return nil
 }
 
+func (config *Config) loadEc2Hosts() ([]string, error) {
+	awsConfig := &aws.Config{
+		Region: aws.String(config.AwsRegion),
+	}
+
+	if config.AwsAccessKey != "" {
+		awsConfig.Credentials = credentials.NewStaticCredentials(config.AwsAccessKey, config.AwsSecretKey, "")
+	}
+
+	svc := ec2.New(session.New(), awsConfig)
+
+	var search bytes.Buffer
+	search.WriteString("tag:")
+	search.WriteString(config.EC2TagKey)
+
+	resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name: aws.String(search.String()),
+				Values: []*string{
+					aws.String(config.EC2TagValue),
+				},
+			},
+		},
+	})
+
+	var servers []string
+	if err != nil {
+		return servers, fmt.Errorf("Unable to fetch EC2 instances: %s", err)
+	}
+
+	for i := range resp.Reservations {
+		for _, inst := range resp.Reservations[i].Instances {
+			servers = append(servers, *inst.PrivateIpAddress)
+		}
+	}
+
+	return servers, nil
+}
+
 // setupLoggers is used to setup the logGate, logWriter, and our logOutput
 func (c *Command) setupLoggers(config *Config) (*GatedWriter, *logWriter, io.Writer) {
 	// Setup logging. First create the gated log writer, which will
@@ -1041,6 +1118,11 @@ Options:
   -atlas-join              Enables auto-joining the Atlas cluster
   -atlas-token=token       Provides the Atlas API token
   -atlas-endpoint=1.2.3.4  The address of the endpoint for Atlas integration.
+  -aws-access-key          AWS access key used to search for instances
+  -aws-secret-key          AWS secret key for aws-acces-key
+  -aws-region              AWS region to search for instances
+  -ec2-tag-key=tag         The EC2 instance tag to filter on for EC2 discover
+  -ec2-tag-value=value     The filter value for ec2-tag-key
   -bootstrap               Sets server to bootstrap mode
   -bind=0.0.0.0            Sets the bind address for cluster communication
   -http-port=8500          Sets the HTTP API port to listen on

command/agent/config.go

@@ -429,6 +429,21 @@ type Config struct {
 	// empty, the defaults from the provider are used.
 	AtlasEndpoint string `mapstructure:"atlas_endpoint"`
 
+	// AwsAccessKey is an AWS IAM key used for discovering EC2 instances
+	AwsAccessKey string `mapstructure:"aws_access_key"`
+
+	// AwsSecretKey is the secret key for AwsAccessKey
+	AwsSecretKey string `mapstructure:aws_secret_key`
+
+	// AwsRegion is the region to attempt to discover instances in
+	AwsRegion string `mapstructure:aws_region`
+
+	// EC2TagKey is the tag applied to EC2 instances to filter on to discover servers
+	EC2TagKey string `mapstructure:"ec2_tag_key"`
+
+	// EC2TagValue is the value of ec2-tag-key to filter for
+	EC2TagValue string `mapstructure:"ec2_tag_value"`
+
 	// AEInterval controls the anti-entropy interval. This is how often
 	// the agent attempts to reconcile its local state with the server's
 	// representation of our state. Defaults to every 60s.
@@ -1259,6 +1274,21 @@ func MergeConfig(a, b *Config) *Config {
 	if b.AtlasEndpoint != "" {
 		result.AtlasEndpoint = b.AtlasEndpoint
 	}
+	if b.AwsAccessKey != "" {
+		result.AwsAccessKey = b.AwsAccessKey
+	}
+	if b.AwsSecretKey != "" {
+		result.AwsSecretKey = b.AwsSecretKey
+	}
+	if b.AwsRegion != "" {
+		result.AwsRegion = b.AwsRegion
+	}
+	if b.EC2TagKey != "" {
+		result.EC2TagKey = b.EC2TagKey
+	}
+	if b.EC2TagValue != "" {
+		result.EC2TagValue = b.EC2TagValue
+	}
 	if b.DisableCoordinates {
 		result.DisableCoordinates = true
 	}