Added getter functionality and dynamic file type handler

Much more work to be done, but using getter will be a helpful improvement as it guarantees that the code base files will be there unless the repo source is invalid. Dynamic file types is the first step in making the code able to handle multiple languages.

.gitignore

@@ -0,0 +1 @@
+.dst/*

main.go

@@ -1,6 +1,9 @@
 package main
 
-import "fmt"
+import (
+	"fmt"
+	"github.com/hashicorp/go-getter"
+)
 
 // TODO add support for `status-react`
 
@@ -17,13 +20,37 @@ const (
 	ignore    = statusDir + "/vendor"
 )
 
+type Repo struct {
+	Dst string
+	Src string
+	FileTypes []string // To be used with (tf TodoFinder) isValidFile
+}
+
+var (
+	repos = []Repo{
+		{"./.dst/status-go", "github.com/status-im/status-go", []string{"go"}},
+		{"./.dst/status-react", "github.com/status-im/status-react", []string{"cljs"}},
+	}
+)
+
 func main() {
-	tf, err := NewTodoFinder()
+	repo := &Repo{
+		Dst:  ".dst/todos-to-docs",
+		Src: "github.com/status-im/todo-to-docs",
+		FileTypes: []string{"go"},
+	}
+
+	err := getter.Get(repo.Dst, repo.Src)
 	if err != nil {
 		panic(err)
 	}
 
-	err = tf.FindInDir(statusDir)
+	tf, err := NewTodoFinder(repo)
+	if err != nil {
+		panic(err)
+	}
+
+	err = tf.Find()
 	if err != nil {
 		panic(err)
 	}

todo_finder.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"regexp"
@@ -22,13 +23,16 @@ type TodoFinder struct {
 	openTodo *todo
 
 	keywords []string
+
+	repo *Repo
 }
 
-func NewTodoFinder() (TodoFinder, error) {
+func NewTodoFinder(repo *Repo) (TodoFinder, error) {
 	tf := TodoFinder{
 		FoundTable: []*todo{},
 		foundTree:  &node{Name: "root", Type: DIR},
 		keywords:   []string{"todo", "fixme"},
+		repo: repo,
 	}
 
 	return tf, tf.init()
@@ -58,6 +62,14 @@ func (tf *TodoFinder) AddTodo(t *todo) {
 	tf.foundTree.AddToTree(t.Path(), t)
 }
 
+func (tf *TodoFinder) Find() error {
+	if tf.repo == nil {
+		return errors.New("repo not set")
+	}
+
+	return tf.FindInDir(tf.repo.Dst)
+}
+
 func (tf *TodoFinder) FindInDir(dir string) error {
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
@@ -78,7 +90,7 @@ func (tf *TodoFinder) FindInDir(dir string) error {
 			}
 		}
 
-		if !tf.isGoFile(f.Name()) {
+		if !tf.isValidFile(f.Name()) {
 			continue
 		}
 
@@ -122,16 +134,23 @@ func (tf *TodoFinder) FindInDir(dir string) error {
 	return nil
 }
 
-func (tf TodoFinder) isGoFile(name string) bool {
-	if len(name) < 3 {
-		return false
+func (tf TodoFinder) isValidFile(name string) bool {
+	valid := false
+	for _, ft := range tf.repo.FileTypes {
+		ftl := len(ft) + 1
+		if len(name) < ftl {
+			return false
+		}
+		last := name[ftl:]
+		valid = last == "."+ft
 	}
-	last := name[len(name)-3:]
-	return last == ".go"
+	return valid
 }
 
 func (tf TodoFinder) buildRegexPattern() string {
 	kwp := tf.makeRegexKeywords()
+	// TODO add functionality for other types of commenting out
+	//  example ';;' or ';' for clojure, '#' and '/* */' blocks
 	return fmt.Sprintf("//.*((%s)(.*))", kwp)
 }
 

todo_finder_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"github.com/davecgh/go-spew/spew"
+	"github.com/hashicorp/go-getter"
 	"testing"
 )
 
@@ -18,3 +19,11 @@ func TestNode_AddToTree(t *testing.T) {
 
 	spew.Dump(n)
 }
+
+// TODO use `getter` to get a local copy of the repo.
+//  getter checks if the local files are upto date with the repo, if not updates otherwise continues
+func TestNewTodoFinder(t *testing.T) {
+	for _, repo := range repos {
+		spew.Dump(getter.Get(repo.Dst, repo.Src))
+	}
+}