From 9ece75fc7825e6aa9f956351e382917871791a4c Mon Sep 17 00:00:00 2001 From: Justin Lambert Date: Wed, 11 May 2016 19:41:41 -0600 Subject: [PATCH] allow automatically discovering aws ec2 consul servers --- command/agent/command.go | 82 ++++++++++++++++++++++++++++++++++++++++ command/agent/config.go | 30 +++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/command/agent/command.go b/command/agent/command.go index b59e0a88de..f57cb63341 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -1,6 +1,7 @@ package agent import ( + "bytes" "flag" "fmt" "io" @@ -15,6 +16,10 @@ import ( "github.com/armon/go-metrics" "github.com/armon/go-metrics/datadog" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/watch" "github.com/hashicorp/go-checkpoint" @@ -94,6 +99,12 @@ func (c *Command) readConfig() *Config { cmdFlags.BoolVar(&cmdConfig.AtlasJoin, "atlas-join", false, "auto-join with Atlas") cmdFlags.StringVar(&cmdConfig.AtlasEndpoint, "atlas-endpoint", "", "endpoint for Atlas integration") + cmdFlags.StringVar(&cmdConfig.AwsAccessKey, "aws-access-key", "", "AWS key for EC2 discovery") + cmdFlags.StringVar(&cmdConfig.AwsSecretKey, "aws-secret-key", "", "AWS secret for EC2 discovery") + cmdFlags.StringVar(&cmdConfig.AwsRegion, "aws-region", "", "Region to search for instances in") + cmdFlags.StringVar(&cmdConfig.EC2TagKey, "ec2-tag-key", "", "EC2 tag key to filter for server discovery") + cmdFlags.StringVar(&cmdConfig.EC2TagValue, "ec2-tag-value", "", "EC2 tag value to filter for server discovery") + cmdFlags.IntVar(&cmdConfig.Protocol, "protocol", -1, "protocol version") cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false, @@ -299,6 +310,32 @@ func (c *Command) readConfig() *Config { c.Ui.Error("WARNING: Bootstrap mode enabled! Do not enable unless necessary") } + if (config.AwsAccessKey != "" || config.AwsSecretKey != "") && (config.AwsAccessKey == "" && config.AwsSecretKey == "") { + c.Ui.Error("aws-acces-key and aws-secret-key are required together") + return nil + } + + if config.EC2TagKey != "" || config.EC2TagValue != "" { + if config.EC2TagKey == "" && config.EC2TagValue == "" { + c.Ui.Error("ec2-tag-key and ec2-tag-value are required together") + return nil + } + + if config.AwsRegion == "" { + c.Ui.Error("aws-region is required") + return nil + } + + ec2servers, err := config.loadEc2Hosts() + if err != nil { + c.Ui.Error(fmt.Sprintf("Unable to query EC2 insances: %s", err)) + return nil + } + config.StartJoin = append(config.StartJoin, ec2servers...) + fmt.Println(config.StartJoin) + os.Exit(1) + } + // Set the version info config.Revision = c.Revision config.Version = c.Version @@ -351,6 +388,46 @@ func (config *Config) verifyUniqueListeners() error { return nil } +func (config *Config) loadEc2Hosts() ([]string, error) { + awsConfig := &aws.Config{ + Region: aws.String(config.AwsRegion), + } + + if config.AwsAccessKey != "" { + awsConfig.Credentials = credentials.NewStaticCredentials(config.AwsAccessKey, config.AwsSecretKey, "") + } + + svc := ec2.New(session.New(), awsConfig) + + var search bytes.Buffer + search.WriteString("tag:") + search.WriteString(config.EC2TagKey) + + resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String(search.String()), + Values: []*string{ + aws.String(config.EC2TagValue), + }, + }, + }, + }) + + var servers []string + if err != nil { + return servers, fmt.Errorf("Unable to fetch EC2 instances: %s", err) + } + + for i := range resp.Reservations { + for _, inst := range resp.Reservations[i].Instances { + servers = append(servers, *inst.PrivateIpAddress) + } + } + + return servers, nil +} + // setupLoggers is used to setup the logGate, logWriter, and our logOutput func (c *Command) setupLoggers(config *Config) (*GatedWriter, *logWriter, io.Writer) { // Setup logging. First create the gated log writer, which will @@ -1041,6 +1118,11 @@ Options: -atlas-join Enables auto-joining the Atlas cluster -atlas-token=token Provides the Atlas API token -atlas-endpoint=1.2.3.4 The address of the endpoint for Atlas integration. + -aws-access-key AWS access key used to search for instances + -aws-secret-key AWS secret key for aws-acces-key + -aws-region AWS region to search for instances + -ec2-tag-key=tag The EC2 instance tag to filter on for EC2 discover + -ec2-tag-value=value The filter value for ec2-tag-key -bootstrap Sets server to bootstrap mode -bind=0.0.0.0 Sets the bind address for cluster communication -http-port=8500 Sets the HTTP API port to listen on diff --git a/command/agent/config.go b/command/agent/config.go index 21a3fa454b..3018bb674a 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -429,6 +429,21 @@ type Config struct { // empty, the defaults from the provider are used. AtlasEndpoint string `mapstructure:"atlas_endpoint"` + // AwsAccessKey is an AWS IAM key used for discovering EC2 instances + AwsAccessKey string `mapstructure:"aws_access_key"` + + // AwsSecretKey is the secret key for AwsAccessKey + AwsSecretKey string `mapstructure:aws_secret_key` + + // AwsRegion is the region to attempt to discover instances in + AwsRegion string `mapstructure:aws_region` + + // EC2TagKey is the tag applied to EC2 instances to filter on to discover servers + EC2TagKey string `mapstructure:"ec2_tag_key"` + + // EC2TagValue is the value of ec2-tag-key to filter for + EC2TagValue string `mapstructure:"ec2_tag_value"` + // AEInterval controls the anti-entropy interval. This is how often // the agent attempts to reconcile its local state with the server's // representation of our state. Defaults to every 60s. @@ -1259,6 +1274,21 @@ func MergeConfig(a, b *Config) *Config { if b.AtlasEndpoint != "" { result.AtlasEndpoint = b.AtlasEndpoint } + if b.AwsAccessKey != "" { + result.AwsAccessKey = b.AwsAccessKey + } + if b.AwsSecretKey != "" { + result.AwsSecretKey = b.AwsSecretKey + } + if b.AwsRegion != "" { + result.AwsRegion = b.AwsRegion + } + if b.EC2TagKey != "" { + result.EC2TagKey = b.EC2TagKey + } + if b.EC2TagValue != "" { + result.EC2TagValue = b.EC2TagValue + } if b.DisableCoordinates { result.DisableCoordinates = true }