sshfp-generator: New features and fixes

* Support for deleted hosts (cleanup records after configured time)
* Small fixes

Signed-off-by: Artur Marud <artur@status.im>

README.md

@@ -16,7 +16,8 @@ go build -mod vendor
 Supported env variables:
 `DOMAIN_NAME` - Domain name which will be working on
 `CF_TOKEN` - CloudFlare Token with write access to above domain
-`CONSUL_TOKEN` - Consul Token with priviledges to read services
+`HOST_LIVENESS_TIMEOUT` - number in seconds after which host is 
+considered as removed and dns records can be deleted
 
 It's possible to create json formatted config file (example in `testcfg`)
 

cloudflare/interface.go

@@ -16,6 +16,7 @@ type Repository interface {
 type Service interface {
 	FindHostByName(hostname string) (bool, error)
 	GetSSHFPRecordsForHost(hostname string) ([]*sshfp.SSHFPRecord, error)
+	DeleteSSHFPRecordsForHost(hostname string) error
 	CreateSSHFPRecord(hostname string, record sshfp.SSHFPRecord) (int, error)
 	DeleteSSHFPRecord(hostname string, record sshfp.SSHFPRecord) error
 	UpdateSSHFPRecord(hostname string, record sshfp.SSHFPRecord) error

cloudflare/service.go

@@ -63,6 +63,23 @@ func (s *service) GetSSHFPRecordsForHost(hostname string) ([]*sshfp.SSHFPRecord,
 	return output, nil
 }
 
+func (s *service) DeleteSSHFPRecordsForHost(hostname string) error {
+	logrus.Debugf("cloudflare: DeleteSSHFPRecordsForHost: %s", hostname)
+	records, err := s.GetSSHFPRecordsForHost(hostname)
+	if err != nil {
+		return err
+	}
+
+	for _, record := range records {
+		err := s.DeleteSSHFPRecord(hostname, *record)
+		if err != nil {
+			return nil
+		}
+	}
+	return nil
+
+}
+
 func (s *service) CreateSSHFPRecord(hostname string, record sshfp.SSHFPRecord) (int, error) {
 	logrus.Infof("cloudflare: CreateSSHFPRecord: %+v", record)
 

config/entity.go

@@ -1,7 +1,9 @@
 package config
 
 type Config struct {
-	ConsulToken     string `json:"consulKey"`
+	//ConsulToken     string `json:"consulKey"`
 	CloudflareToken string `json:"cloudflareKey"`
 	DomainName      string `json:"domain"`
+	HostTimeout     int    `json:"hostLivenessTimeout"`
+	LogLevel        string `json:"logLevel"`
 }

config/service.go

@@ -3,6 +3,7 @@ package config
 import (
 	"errors"
 	"os"
+	"strconv"
 
 	"github.com/sirupsen/logrus"
 )
@@ -22,17 +23,27 @@ func (s *service) LoadConfig(fileName string) (*Config, error) {
 		if !exists {
 			return nil, errors.New("cannot find env variable CF_TOKEN")
 		}
-		consulToken, exists := os.LookupEnv("CONSUL_TOKEN")
-		if !exists {
-			return nil, errors.New("cannot find env variable CONSUL_TOKEN")
-		}
-
 		domainaName, exists := os.LookupEnv("DOMAIN_NAME")
-		if exists {
+		if !exists {
 			return nil, errors.New("cannot find env variable DOMAIN_NAME")
 		}
 
-		return &Config{ConsulToken: consulToken, CloudflareToken: cfToken, DomainName: domainaName}, nil
+		hostTimeout, exists := os.LookupEnv("HOST_LIVENESS_TIMEOUT")
+		if !exists {
+			return nil, errors.New("cannot find env variable HOST_LIVENESS_TIMEOUT")
+		}
+		hostTimeoutInt, err := strconv.ParseInt(hostTimeout, 10, 32)
+		if err != nil {
+			return nil, errors.New("incorrect HOST_LIVENESS_TIMEOUT value")
+
+		}
+
+		logLevel, exists := os.LookupEnv(("LOG_LEVEL"))
+		if !exists {
+			return nil, errors.New("cannot find env variable LOG_LEVEL")
+		}
+
+		return &Config{CloudflareToken: cfToken, DomainName: domainaName, HostTimeout: int(hostTimeoutInt), LogLevel: logLevel}, nil
 
 	}
 

main.go

@@ -6,17 +6,24 @@ import (
 	"infra-sshfp-cf/consul"
 	"infra-sshfp-cf/sshfp"
 	"infra-sshfp-cf/statestore"
+	"os"
 
 	"github.com/sirupsen/logrus"
 )
 
 func main() {
 	//Debug loglevel
-	logrus.SetLevel(logrus.InfoLevel)
+	logrus.SetLevel(logrus.DebugLevel)
 
 	//Create configuration components
+	// Get config file name from args, if empty - try to configure from ENVs
+	var configFilename string = ""
+	if len(os.Args) > 1 {
+		configFilename = os.Args[1]
+	}
+
 	cfgService := config.NewService(config.NewFileRepository())
-	config, err := cfgService.LoadConfig("testcfg")
+	config, err := cfgService.LoadConfig(configFilename)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -81,4 +88,15 @@ func main() {
 		}
 	}
 
+	//Purge old hosts
+	hosts, _ = statestore.GetStalledHosts(config.HostTimeout)
+	for _, host := range hosts {
+		err := cloudflare.DeleteSSHFPRecordsForHost(host)
+		if err != nil {
+			logrus.Fatalf("Cannot delete records for host: %s", host)
+		}
+
+	}
+	statestore.PurgeStalledHosts(config.HostTimeout)
+
 }

statestore/interface.go

@@ -1,11 +1,18 @@
 package statestore
 
+import "time"
+
 type Repository interface {
 	GetModifyIndex(hostname string) (int, error)
 	SetModifyIndex(hostname string, index int) error
+	DeleteHost(hostname string) error
+	DeleteHosts(hostnames []string) error
+	GetOutdatedHosts(time time.Duration) ([]string, error)
 }
 
 type Service interface {
 	CheckIfModified(hostname string, index int) (bool, error)
 	SaveState(hostname string, index int) error
+	PurgeStalledHosts(timeTreshold int) error
+	GetStalledHosts(timeTreshold int) ([]string, error)
 }

statestore/mapRepository.go

@@ -3,12 +3,18 @@ package statestore
 import (
 	"encoding/json"
 	"io/ioutil"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
 
+type mapEntry struct {
+	ModifyIndex int
+	LastSeen    time.Time
+}
+
 type mapRepository struct {
-	db       map[string]int
+	db       map[string]mapEntry
 	filename string
 }
 
@@ -21,7 +27,7 @@ func NewMapRepository(filename string) Repository {
 		return repo
 	}
 
-	db := make(map[string]int)
+	db := make(map[string]mapEntry)
 	repo.db = db
 
 	err = repo.saveDatabase()
@@ -34,16 +40,53 @@ func NewMapRepository(filename string) Repository {
 
 func (r *mapRepository) GetModifyIndex(hostname string) (int, error) {
 	if value, ok := r.db[hostname]; ok {
-		return value, nil
+		r.SetModifyIndex(hostname, value.ModifyIndex)
+		return value.ModifyIndex, nil
 	}
 	return -1, nil
 }
 
+func (r *mapRepository) GetOutdatedHosts(duration time.Duration) ([]string, error) {
+	var output []string
+	for k, v := range r.db {
+
+		if v.LastSeen.Add(duration).Before(time.Now()) {
+			output = append(output, k)
+		}
+
+	}
+	return output, nil
+}
+
 func (r *mapRepository) SetModifyIndex(hostname string, index int) error {
-	r.db[hostname] = index
+	if value, ok := r.db[hostname]; ok {
+		value.LastSeen = time.Now()
+		value.ModifyIndex = index
+		r.db[hostname] = value
+	} else {
+		value := new(mapEntry)
+		value.LastSeen = time.Now()
+		value.ModifyIndex = index
+		r.db[hostname] = *value
+	}
 	return r.saveDatabase()
 }
 
+func (r *mapRepository) DeleteHost(hostname string) error {
+
+	logrus.Debugf("mapRepository: DeleteHost %s", hostname)
+
+	delete(r.db, hostname)
+	return r.saveDatabase()
+}
+
+func (r *mapRepository) DeleteHosts(hostnames []string) error {
+	for _, host := range hostnames {
+		r.DeleteHost(host)
+	}
+	return nil
+}
+
 func (r *mapRepository) openDatabase() error {
 	logrus.Infof("mapRepository: openDatabase %s", r.filename)
 	content, err := ioutil.ReadFile(r.filename)

statestore/service.go

@@ -1,6 +1,10 @@
 package statestore
 
-import "github.com/sirupsen/logrus"
+import (
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
 
 type service struct {
 	r Repository
@@ -30,3 +34,23 @@ func (s *service) SaveState(hostname string, index int) error {
 	return s.r.SetModifyIndex(hostname, index)
 
 }
+
+func (s *service) GetStalledHosts(timeTreshold int) ([]string, error) {
+	return s.r.GetOutdatedHosts(time.Duration(timeTreshold) * time.Second)
+
+}
+
+func (s *service) PurgeStalledHosts(timeTreshold int) error {
+	logrus.Debugf("statestore: PurgeStalledHosts: %d", timeTreshold)
+	hosts, err := s.r.GetOutdatedHosts(time.Duration(timeTreshold) * time.Second)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("statestore: PurgeStalledHosts: %+v", hosts)
+
+	s.r.DeleteHosts(hosts)
+
+	return nil
+
+}