Repository: bitfield/script
Pull request #36 (2019-10-30 22:08:00)

Add Findfiles func to find files recursively
Add a `FindFiles` func that will find files recursively in all directories (#35)

**NOTE:** 

I wanted the method to accept glob, but its behaving strangely in go. When you pass `dir/**/*.txt`, it only finds files inside the subdirectories ignoring all files in the root directory. 

bitfield: Are you ready for a review yet @mainawycliffe or are you still working on it?
mainawycliffe: I am ready for a review. I think we can add glob to `FindFile` func if the situation changes in the future.
mainawycliffe: I am Sorry, I accidentally merged my branch on #23 and became too tangled to undo. I think the [example](https://github.com/mainawycliffe/script/tree/findfiles/examples/todo) is also for a review, so you can take a look at that too.
mainawycliffe: I have created a new repo for the find Todos program [here](https://github.com/mainawycliffe/script-find-todos). This is actually a good idea, as I can work to improve it and improve comment detection on the repo. 
mainawycliffe: no problem @bitfield, that happens all the time.
mainawycliffe: Taken care of the changes requested above.
bitfield: Great work, @mainawycliffe! Do you mind if I give this a final pass for style, consistency, squash commits, etc?
mainawycliffe: No, I don't. 

Patch:
diff --git a/README.md b/README.md
index d8b143f..ab793ac 100644
--- a/README.md
+++ b/README.md
@@ -91,6 +91,7 @@ John is also a [Kubernetes and cloud infrastructure consultant](https://bitfield
 		- [Error output](#error-output)
 	- [File](#file)
 	- [IfExists](#ifexists)
+	- [FindFiles](#findfiles)
 	- [ListFiles](#listfiles)
 	- [Slice](#slice)
 	- [Stdin](#stdin)
@@ -299,6 +300,7 @@ If you're already familiar with shell scripting and the Unix toolset, here is a
 | `grep`             | [`Match()`](#match) / [`MatchRegexp()`](#matchregexp)         |
 | `grep -v`          | [`Reject()`](#reject) / [`RejectRegexp()`](#rejectregexp)     |
 | `head`             | [`First()`](#first)                                           |
+| `find`             | [`FindFiles`](#findfiles)                                     |
 | `ls`               | [`ListFiles()`](#listfiles)                                   |
 | `sed`              | [`Replace()`](#replace) / [`ReplaceRegexp()`](#replaceregexp) |
 | `tail`             | [`Last()`](#last)                                             |
@@ -418,6 +420,15 @@ This can be used to create pipes which take some action only if a certain file e
 script.IfExists("/foo/bar").Exec("/usr/bin/yada")
 ```
 
+## FindFiles
+
+`FindFiles()` lists all files in a directory and its subdirectories recursively, like Unix [`find -type f`](examples/find/main.go).
+
+```go
+script.FindFiles("/tmp").Stdout()
+// lists all files in /tmp and its subtrees
+```
+
 ## ListFiles
 
 `ListFiles()` lists files, like Unix [`ls`](examples/ls/main.go). It creates a pipe containing all files and directories matching the supplied path specification, one per line. This can be the name of a directory (`/path/to/dir`), the name of a file (`/path/to/file`), or a _glob_ (wildcard expression) conforming to the syntax accepted by [filepath.Match()](https://golang.org/pkg/path/filepath/#Match) (`/path/to/*`).
diff --git a/examples/find/main.go b/examples/find/main.go
new file mode 100644
index 0000000..cfc2b32
--- /dev/null
+++ b/examples/find/main.go
@@ -0,0 +1,15 @@
+package main
+
+import (
+	"os"
+
+	"github.com/bitfield/script"
+)
+
+func main() {
+	listPath := "."
+	if len(os.Args) > 1 {
+		listPath = os.Args[1]
+	}
+	script.FindFiles(listPath).Stdout()
+}
diff --git a/examples/todo/examples.text b/examples/todo/examples.text
new file mode 100644
index 0000000..5e1fdbd
--- /dev/null
+++ b/examples/todo/examples.text
@@ -0,0 +1,94 @@
+package script
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"regexp"
+)
+
+type Todo struct {
+	LineNumber int32
+	Message    string
+	FilePath   string
+}
+/*** 
+todo 1
+todo 2
+todo 3
+*/
+
+/** todo ddjdjdj */
+
+# todo say something
+// todo hello world
+// FindTodos Open file and find todos
+func FindTodos(filePath string) (*[]Todo, error) {
+
+	file, err := os.Open(filePath)
+
+	defer file.Close()
+
+	if err != nil {
+		return nil, fmt.Errorf("Could not open file %s: %v", filePath, err)
+	}
+
+	scanner := bufio.NewScanner(file)
+	scanner.Split(bufio.ScanLines)
+
+	lineNumber := 1
+
+	foundTodos := make([]Todo, 0)
+
+	for scanner.Scan() {
+		// find todos in every line
+		// ideally, the todo will be inside a comment bloc i.e. // /**/ ///
+		findTodo, err := regexp.Match(`((?i)\/\/|#).todo.*`, scanner.Bytes()) ///// todo another todo here
+
+		if err != nil {
+			return nil, fmt.Errorf("An error occurred while finding todo: %v", err)
+		}
+
+		if findTodo == true {
+
+			todo := Todo{
+				LineNumber: int32(lineNumber),
+				Message:    scanner.Text(),
+				FilePath:   filePath,
+			}
+
+			foundTodos = append(foundTodos, todo)
+
+			fmt.Printf("%d: %s\n", lineNumber, scanner.Text())
+
+		}
+
+		lineNumber++
+	}
+
+	return &foundTodos, nil
+}
+
+// Opts Options for getting files in a directory/subdirectory
+type Opts struct {
+	RespectGitignore bool
+	SkipHiddenFiles  bool
+}
+
+// GetFilesInDirectory scan directory and subdirectories, returning all files
+// except those that are exempted. Exempted files could be either by an exact match or
+// a regex match. Directories can also be exempted.
+func GetFilesInDirectory(opts Opts, exempt ...string) ([]string, error) {
+	// TODO read all files and directories inside
+	// TODO have a list of files to exempt, either by
+	// todos: extension, filename or regex pattern
+	// todo: for each file, check if there are any todos
+
+	return nil, nil
+
+}
+
+// GetGitingoreFiles find all gitingore files and return files to ignore
+func GetGitingoreFiles() ([]string, error) {
+	return nil, nil
+}
diff --git a/examples/todo/main.go b/examples/todo/main.go
new file mode 100644
index 0000000..d005975
--- /dev/null
+++ b/examples/todo/main.go
@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/bitfield/script"
+)
+
+func main() {
+	listPath := "."
+	if len(os.Args) > 1 {
+		listPath = os.Args[1]
+	}
+	// filter hidden directories and files
+	filterFiles := regexp.MustCompile(`^\..*|/\.`)
+	files := script.FindFiles(listPath).RejectRegexp(filterFiles)
+	content := files.EachLine(func(filePath string, builderFile *strings.Builder) {
+		p := script.File(filePath)
+		lineNumber := 1
+		p.EachLine(func(str string, build *strings.Builder) {
+			if strings.Contains(str, "todo") {
+				builderFile.WriteString(fmt.Sprintf("%s:%d %s \n", filePath, lineNumber, strings.TrimSpace(str)))
+			}
+			lineNumber++
+		})
+	})
+	content.Stdout()
+}
diff --git a/sources.go b/sources.go
index 1fe7810..8cdb6ef 100644
--- a/sources.go
+++ b/sources.go
@@ -53,6 +53,27 @@ func File(name string) *Pipe {
 	return p.WithReader(f)
 }
 
+// FindFiles takes a directory path and returns a pipe listing all the files in
+// the directory and its subdirectories recursively, one per line, like Unix
+// `find -type f`. If the path doesn't exist or can't be read, the pipe's error
+// status will be set.
+func FindFiles(path string) *Pipe {
+	var fileNames []string
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			fileNames = append(fileNames, path)
+		}
+		return nil
+	}
+	if err := filepath.Walk(path, walkFn); err != nil {
+		return NewPipe().WithError(err)
+	}
+	return Slice(fileNames)
+}
+
 // ListFiles creates a pipe containing the files and directories matching the
 // supplied path, one per line. The path may be a glob, conforming to
 // filepath.Match syntax.


Test patch:
diff --git a/sources_test.go b/sources_test.go
index 6af612f..177f4ac 100644
--- a/sources_test.go
+++ b/sources_test.go
@@ -127,6 +127,47 @@ func TestFile(t *testing.T) {
 	}
 }
 
+func TestFindFiles(t *testing.T) {
+	tcs := []struct {
+		Path           string
+		ErrExpected    bool
+		wantErrContain string
+		Want           string
+	}{
+		{
+			Path:        "testdata/multiple_files",
+			ErrExpected: false,
+			Want:        "testdata/multiple_files/1.txt\ntestdata/multiple_files/2.txt\ntestdata/multiple_files/3.tar.zip\n",
+		},
+		{
+			Path:        "testdata/multiple_files_with_subdirectory",
+			ErrExpected: false,
+			Want:        "testdata/multiple_files_with_subdirectory/1.txt\ntestdata/multiple_files_with_subdirectory/2.txt\ntestdata/multiple_files_with_subdirectory/3.tar.zip\ntestdata/multiple_files_with_subdirectory/dir/.hidden\ntestdata/multiple_files_with_subdirectory/dir/1.txt\ntestdata/multiple_files_with_subdirectory/dir/2.txt\n",
+		},
+		{
+			Path:        "noneexistentpath",
+			ErrExpected: true,
+			Want:        "",
+		},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.Path, func(t *testing.T) {
+			p := FindFiles(tc.Path)
+			if tc.ErrExpected != (p.Error() != nil) {
+				t.Fatalf("unexpected error value: %v", p.Error())
+			}
+			p.SetError(nil) // else p.String() would be a no-op
+			got, err := p.String()
+			if err != nil {
+				t.Fatalf("unexpected error %v", err)
+			}
+			if !cmp.Equal(tc.Want, got) {
+				t.Fatalf("want %q, got %q", tc.Want, got)
+			}
+		})
+	}
+}
+
 func TestIfExists(t *testing.T) {
 	t.Parallel()
 	p := IfExists("testdata/doesntexist")