Repository: diggerhq/digger
Pull request #444 (2023-07-12 15:05:02)

add auto-detection of projects in case of missing digger.yml
https://github.com/diggerhq/digger/issues/443

veziak: I would move auto detection tests to a separate file, because it's important feature that probably will be improved and grow a bit more

Patch:
diff --git a/cmd/digger/main.go b/cmd/digger/main.go
index e58957f2..eee455c5 100644
--- a/cmd/digger/main.go
+++ b/cmd/digger/main.go
@@ -60,9 +60,7 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker, reporti
 	}
 	println("GitHub context parsed successfully")
 
-	walker := configuration.FileSystemDirWalker{}
-
-	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig("./", &walker)
+	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig("./")
 	if err != nil {
 		reportErrorAndExit(githubActor, fmt.Sprintf("Failed to read Digger config. %s", err), 4)
 	}
@@ -148,14 +146,13 @@ func gitLabCI(lock core_locking.Lock, policyChecker core_policy.Checker, reporti
 		fmt.Println("GITLAB_TOKEN is empty")
 	}
 
-	walker := configuration.FileSystemDirWalker{}
 	currentDir, err := os.Getwd()
 	if err != nil {
 		reportErrorAndExit(projectNamespace, fmt.Sprintf("Failed to get current dir. %s", err), 4)
 	}
 	fmt.Printf("main: working dir: %s \n", currentDir)
 
-	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig(currentDir, &walker)
+	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig(currentDir)
 	if err != nil {
 		reportErrorAndExit(projectNamespace, fmt.Sprintf("Failed to read Digger config. %s", err), 4)
 	}
@@ -243,14 +240,13 @@ func azureCI(lock core_locking.Lock, policyChecker core_policy.Checker, reportin
 		os.Exit(4)
 	}
 
-	walker := configuration.FileSystemDirWalker{}
 	currentDir, err := os.Getwd()
 	if err != nil {
 		reportErrorAndExit(parsedAzureContext.BaseUrl, fmt.Sprintf("Failed to get current dir. %s", err), 4)
 	}
 	fmt.Printf("main: working dir: %s \n", currentDir)
 
-	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig(currentDir, &walker)
+	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig(currentDir)
 	if err != nil {
 		reportErrorAndExit(parsedAzureContext.BaseUrl, fmt.Sprintf("Failed to read Digger config. %s", err), 4)
 	}
diff --git a/go.mod b/go.mod
index 2e9a16f8..452130c0 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	cloud.google.com/go/storage v1.30.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1
-	github.com/aws/aws-sdk-go v1.44.296
+	github.com/aws/aws-sdk-go v1.44.300
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/caarlos0/env/v8 v8.0.0
 	github.com/google/go-github/v53 v53.2.0
diff --git a/go.sum b/go.sum
index f3e0028b..a504d38f 100644
--- a/go.sum
+++ b/go.sum
@@ -39,6 +39,8 @@ github.com/aws/aws-sdk-go v1.44.295 h1:SGjU1+MqttXfRiWHD6WU0DRhaanJgAFY+xIhEaugV
 github.com/aws/aws-sdk-go v1.44.295/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go v1.44.296 h1:ALRZIIKI+6EBWDiWP4RHWmOtHZ7dywRzenL4NWgNI2A=
 github.com/aws/aws-sdk-go v1.44.296/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY=
+github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=
diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go
index 1e67bc60..ff550b04 100644
--- a/pkg/configuration/converters.go
+++ b/pkg/configuration/converters.go
@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 )
 
+const defaultWorkflowName = "default"
+
 func copyProjects(projects []*ProjectYaml) []Project {
 	result := make([]Project, len(projects))
 	for i, p := range projects {
@@ -101,9 +103,8 @@ func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]Workflow {
 	return result
 }
 
-func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, graph.Graph[string, string], error) {
+func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string) (*DiggerConfig, graph.Graph[string, string], error) {
 	var diggerConfig DiggerConfig
-	const defaultWorkflowName = "default"
 
 	if diggerYaml.AutoMerge != nil {
 		diggerConfig.AutoMerge = *diggerYaml.AutoMerge
@@ -162,7 +163,13 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string,
 	}
 
 	if diggerYaml.GenerateProjectsConfig != nil {
-		dirs, err := walker.GetDirs(workingDir)
+		var dirWalker DirWalker
+		if diggerYaml.GenerateProjectsConfig.Terragrunt {
+			dirWalker = &FileSystemTerragruntDirWalker{}
+		} else {
+			dirWalker = &FileSystemTopLevelTerraformDirWalker{}
+		}
+		dirs, err := dirWalker.GetDirs(workingDir)
 		if err != nil {
 			return nil, nil, err
 		}
diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go
index fc07b641..d2210dd1 100644
--- a/pkg/configuration/digger_config.go
+++ b/pkg/configuration/digger_config.go
@@ -6,17 +6,25 @@ import (
 	"fmt"
 	"github.com/dominikbraun/graph"
 	"gopkg.in/yaml.v3"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
+	"strings"
 )
 
 type DirWalker interface {
 	GetDirs(workingDir string) ([]string, error)
 }
 
-type FileSystemDirWalker struct {
+type FileSystemTopLevelTerraformDirWalker struct {
+}
+
+type FileSystemTerragruntDirWalker struct {
+}
+
+type FileSystemModuleDirWalker struct {
 }
 
 func GetFilesWithExtension(workingDir string, ext string) ([]string, error) {
@@ -37,7 +45,7 @@ func GetFilesWithExtension(workingDir string, ext string) ([]string, error) {
 	return files, nil
 }
 
-func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) {
+func (walker *FileSystemTopLevelTerraformDirWalker) GetDirs(workingDir string) ([]string, error) {
 	var dirs []string
 	err := filepath.Walk(workingDir,
 		func(path string, info os.FileInfo, err error) error {
@@ -46,9 +54,12 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error)
 				return err
 			}
 			if info.IsDir() {
+				if info.Name() == "modules" {
+					return filepath.SkipDir
+				}
 				terraformFiles, _ := GetFilesWithExtension(path, ".tf")
 				if len(terraformFiles) > 0 {
-					dirs = append(dirs, path)
+					dirs = append(dirs, strings.ReplaceAll(path, workingDir+string(os.PathSeparator), ""))
 					return filepath.SkipDir
 				}
 			}
@@ -60,9 +71,64 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error)
 	return dirs, nil
 }
 
+func (walker *FileSystemModuleDirWalker) GetDirs(workingDir string) ([]string, error) {
+	var dirs []string
+	err := filepath.Walk(workingDir,
+		func(path string, info os.FileInfo, err error) error {
+
+			if err != nil {
+				return err
+			}
+			if info.IsDir() && info.Name() == "modules" {
+				dirs = append(dirs, strings.ReplaceAll(path, workingDir+string(os.PathSeparator), ""))
+				return filepath.SkipDir
+			}
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+	return dirs, nil
+}
+
+func (walker *FileSystemTerragruntDirWalker) GetDirs(workingDir string) ([]string, error) {
+	var dirs []string
+	err := filepath.Walk(workingDir,
+		func(path string, info os.FileInfo, err error) error {
+
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				if info.Name() == "modules" {
+					return filepath.SkipDir
+				}
+				terragruntFiles, _ := GetFilesWithExtension(path, "terragrunt.hcl")
+				if len(terragruntFiles) > 0 {
+					for _, f := range terragruntFiles {
+						terragruntFile := path + string(os.PathSeparator) + f
+						fileContent, err := os.ReadFile(terragruntFile)
+						if err != nil {
+							return err
+						}
+						if strings.Contains(string(fileContent), "include \"root\"") {
+							dirs = append(dirs, strings.ReplaceAll(path, workingDir+string(os.PathSeparator), ""))
+							return filepath.SkipDir
+						}
+					}
+				}
+			}
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+	return dirs, nil
+}
+
 var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'")
 
-func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, graph.Graph[string, string], error) {
+func LoadDiggerConfig(workingDir string) (*DiggerConfig, graph.Graph[string, string], error) {
 	configYaml := &DiggerConfigYaml{}
 	config := &DiggerConfig{}
 	fileName, err := retrieveConfigFile(workingDir)
@@ -73,31 +139,32 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, graph
 	}
 
 	if fileName == "" {
-		fmt.Println("No digger config found, using default one")
-		config.Projects = make([]Project, 1)
-		project := defaultProject()
-		config.Projects[0] = project
-		config.Workflows = make(map[string]Workflow)
-		config.Workflows["default"] = *defaultWorkflow()
-		g := graph.New(graph.StringHash, graph.PreventCycles(), graph.Directed())
-		g.AddVertex(project.Name)
-		return config, g, nil
-	}
-
-	data, err := os.ReadFile(fileName)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read config file %s: %v", fileName, err)
-	}
+		configYaml, err = AutoDetectDiggerConfig(workingDir)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to auto detect digger config: %v", err)
+		}
+		marshalledConfig, err := yaml.Marshal(configYaml)
+		if err != nil {
+			log.Printf("failed to marshal auto detected digger config: %v", err)
+		} else {
+			log.Printf("Auto detected digger config: \n%v", marshalledConfig)
+		}
+	} else {
+		data, err := os.ReadFile(fileName)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to read config file %s: %v", fileName, err)
+		}
 
-	if err := yaml.Unmarshal(data, configYaml); err != nil {
-		return nil, nil, fmt.Errorf("error parsing '%s': %v", fileName, err)
+		if err := yaml.Unmarshal(data, configYaml); err != nil {
+			return nil, nil, fmt.Errorf("error parsing '%s': %v", fileName, err)
+		}
 	}
 
 	if (configYaml.Projects == nil || len(configYaml.Projects) == 0) && configYaml.GenerateProjectsConfig == nil {
 		return nil, nil, fmt.Errorf("no projects configuration found in '%s'", fileName)
 	}
 
-	config, projectDependencyGraph, err := ConvertDiggerYamlToConfig(configYaml, workingDir, walker)
+	config, projectDependencyGraph, err := ConvertDiggerYamlToConfig(configYaml, workingDir)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -111,13 +178,52 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, graph
 	return config, projectDependencyGraph, nil
 }
 
-func defaultProject() Project {
-	return Project{
-		Name:       "default",
-		Dir:        ".",
-		Workspace:  "default",
-		Terragrunt: false,
-		Workflow:   "default",
+func AutoDetectDiggerConfig(workingDir string) (*DiggerConfigYaml, error) {
+	configYaml := &DiggerConfigYaml{}
+	collectUsageData := true
+	configYaml.CollectUsageData = &collectUsageData
+
+	terragruntDirWalker := &FileSystemTerragruntDirWalker{}
+	terraformDirWalker := &FileSystemTopLevelTerraformDirWalker{}
+	moduleDirWalker := &FileSystemModuleDirWalker{}
+
+	terragruntDirs, err := terragruntDirWalker.GetDirs(workingDir)
+
+	if err != nil {
+		return nil, err
+	}
+
+	terraformDirs, err := terraformDirWalker.GetDirs(workingDir)
+
+	if err != nil {
+		return nil, err
+	}
+
+	moduleDirs, err := moduleDirWalker.GetDirs(workingDir)
+
+	modulePatterns := []string{}
+	for _, dir := range moduleDirs {
+		modulePatterns = append(modulePatterns, dir+"/**")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	if len(terragruntDirs) > 0 {
+		// TODO: add support for dependency graph when parsing terragrunt config
+		for _, dir := range terragruntDirs {
+			project := ProjectYaml{Name: dir, Dir: dir, Workflow: defaultWorkflowName, Workspace: "default", Terragrunt: true, IncludePatterns: modulePatterns}
+			configYaml.Projects = append(configYaml.Projects, &project)
+		}
+		return configYaml, nil
+	} else if len(terraformDirs) > 0 {
+		for _, dir := range terraformDirs {
+			project := ProjectYaml{Name: dir, Dir: dir, Workflow: defaultWorkflowName, Workspace: "default", Terragrunt: false, IncludePatterns: modulePatterns}
+			configYaml.Projects = append(configYaml.Projects, &project)
+		}
+		return configYaml, nil
+	} else {
+		return nil, fmt.Errorf("no terragrunt or terraform project detected in the repository")
 	}
 }
 
diff --git a/pkg/configuration/yaml.go b/pkg/configuration/yaml.go
index 2d26e161..2a9069dd 100644
--- a/pkg/configuration/yaml.go
+++ b/pkg/configuration/yaml.go
@@ -68,8 +68,9 @@ type EnvVarYaml struct {
 }
 
 type GenerateProjectsConfigYaml struct {
-	Include string `yaml:"include"`
-	Exclude string `yaml:"exclude"`
+	Include    string `yaml:"include"`
+	Exclude    string `yaml:"exclude"`
+	Terragrunt bool   `yaml:"terragrunt"`
 }
 
 func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error {
diff --git a/pkg/usage/usage.go b/pkg/usage/usage.go
index b48c6348..fa0d65b6 100644
--- a/pkg/usage/usage.go
+++ b/pkg/usage/usage.go
@@ -76,8 +76,7 @@ func init() {
 	if err != nil {
 		fmt.Printf("Failed to get current dir. %s", err)
 	}
-	walker := configuration.FileSystemDirWalker{}
-	config, _, err := configuration.LoadDiggerConfig(currentDir, &walker)
+	config, _, err := configuration.LoadDiggerConfig(currentDir)
 	if err != nil {
 		return
 	}


Test patch:
diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go
index 0496a7a8..b60b7d92 100644
--- a/pkg/configuration/digger_config_test.go
+++ b/pkg/configuration/digger_config_test.go
@@ -11,15 +11,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-type MockDirWalker struct {
-	Files []string
-}
-
-func (walker *MockDirWalker) GetDirs(workingDir string) ([]string, error) {
-
-	return walker.Files, nil
-}
-
 func setUp() (string, func()) {
 	tempDir := createTempDir()
 	return tempDir, func() {
@@ -27,13 +18,6 @@ func setUp() (string, func()) {
 	}
 }
 
-func TestDiggerConfigFileDoesNotExist(t *testing.T) {
-	dg, _, err := LoadDiggerConfig("", &FileSystemDirWalker{})
-	assert.NoError(t, err, "expected error to be not nil")
-	assert.Equal(t, dg.Projects[0].Name, "default", "expected default project to have name 'default'")
-	assert.Equal(t, dg.Projects[0].Dir, ".", "expected default project dir to be '.'")
-}
-
 func TestDiggerConfigWhenMultipleConfigExist(t *testing.T) {
 	tempDir, teardown := setUp()
 	defer teardown()
@@ -48,7 +32,7 @@ func TestDiggerConfigWhenMultipleConfigExist(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.Error(t, err, "expected error to be returned")
 	assert.ErrorContains(t, err, ErrDiggerConfigConflict.Error(), "expected error to match target error")
 	assert.Nil(t, dg, "expected diggerConfig to be nil")
@@ -68,7 +52,7 @@ projects:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
 	assert.Equal(t, "path/to/module/test", dg.GetDirectory("prod"))
@@ -88,7 +72,7 @@ projects:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
@@ -123,7 +107,7 @@ projects:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
 	assert.Equal(t, "default", dg.Projects[0].Workflow)
@@ -145,7 +129,7 @@ projects:
 	deleteFile := createFile(path.Join(tempDir, "digger.yml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
 	assert.Equal(t, "path/to/module", dg.GetDirectory("dev"))
@@ -170,7 +154,7 @@ workflows:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.Equal(t, models.Step{Action: "run", Value: "echo \"hello\"", Shell: ""}, dg.Workflows["myworkflow"].Plan.Steps[0], "parsed struct does not match expected struct")
 }
@@ -215,7 +199,7 @@ workflows:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.Equal(t, []EnvVar{
 		{Name: "TF_VAR_state", Value: "s3://mybucket/terraform.tfstate"},
@@ -255,7 +239,7 @@ workflows:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.Equal(t, models.Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["dev"].Plan.Steps[0], "parsed struct does not match expected struct")
 	assert.Equal(t, models.Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["dev"].Plan.Steps[1], "parsed struct does not match expected struct")
@@ -277,15 +261,15 @@ generate_projects:
 `
 	deleteFile := createFile(path.Join(tempDir, "digger.yml"), diggerCfg)
 	defer deleteFile()
+	dirsToCreate := []string{"dev/test1", "dev/test2", "dev/project", "testtt"}
 
-	walker := &MockDirWalker{}
-	walker.Files = append(walker.Files, "dev")
-	walker.Files = append(walker.Files, "dev/test1")
-	walker.Files = append(walker.Files, "dev/test2")
-	walker.Files = append(walker.Files, "dev/project")
-	walker.Files = append(walker.Files, "testtt")
+	for _, dir := range dirsToCreate {
+		err := os.MkdirAll(path.Join(tempDir, dir), os.ModePerm)
+		defer createFile(path.Join(tempDir, dir, "main.tf"), "")()
+		assert.NoError(t, err, "expected error to be nil")
+	}
 
-	dg, _, err := LoadDiggerConfig(tempDir, walker)
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
 	assert.Equal(t, "test1", dg.Projects[0].Name)
@@ -295,32 +279,66 @@ generate_projects:
 	assert.Equal(t, 2, len(dg.Projects))
 }
 
+func TestGenerateProjectsWithoutDiggerConfig(t *testing.T) {
+	tempDir, teardown := setUp()
+	defer teardown()
+
+	dirsWithTfToCreate := []string{"dev/test1", "dev/test1/db", "dev/test1/vpc", "dev/test2", "dev/test2/db", "dev/test2/vpc", "dev/project", "prod/test1", "prod/test2", "prod/project", "test", "modules/test1", "modules/test2"}
+
+	for _, dir := range dirsWithTfToCreate {
+		err := os.MkdirAll(path.Join(tempDir, dir), os.ModePerm)
+		defer createFile(path.Join(tempDir, dir, "main.tf"), "")()
+		assert.NoError(t, err, "expected error to be nil")
+	}
+
+	dirtsWithoutTfToCreate := []string{"docs", "random", "docs/random"}
+	for _, dir := range dirtsWithoutTfToCreate {
+		err := os.MkdirAll(path.Join(tempDir, dir), os.ModePerm)
+		assert.NoError(t, err, "expected error to be nil")
+	}
+
+	dg, _, err := LoadDiggerConfig(tempDir)
+	assert.NoError(t, err, "expected error to be nil")
+	assert.NotNil(t, dg, "expected digger config to be not nil")
+	assert.Equal(t, "dev/project", dg.Projects[0].Name)
+	assert.Equal(t, "dev/test1", dg.Projects[1].Name)
+	assert.Equal(t, "dev/test2", dg.Projects[2].Name)
+	assert.Equal(t, "prod/project", dg.Projects[3].Name)
+	assert.Equal(t, "prod/test1", dg.Projects[4].Name)
+	assert.Equal(t, "prod/test2", dg.Projects[5].Name)
+	assert.Equal(t, "test", dg.Projects[6].Name)
+	assert.Equal(t, 7, len(dg.Projects))
+}
+
 func TestDiggerGenerateProjectsWithSubDirs(t *testing.T) {
 	tempDir, teardown := setUp()
 	defer teardown()
 
 	diggerCfg := `
 generate_projects:
-  include: dev/*
+  include: dev/**
   exclude: dev/project
 `
 	deleteFile := createFile(path.Join(tempDir, "digger.yml"), diggerCfg)
 	defer deleteFile()
+	dirsToCreate := []string{
+		"dev/test1/utils",
+		"dev/test2",
+		"dev/project",
+		"testtt",
+	}
+	for _, dir := range dirsToCreate {
+		err := os.MkdirAll(path.Join(tempDir, dir), os.ModePerm)
+		defer createFile(path.Join(tempDir, dir, "main.tf"), "")()
+		assert.NoError(t, err, "expected error to be nil")
+	}
 
-	walker := &MockDirWalker{}
-	walker.Files = append(walker.Files, "dev")
-	walker.Files = append(walker.Files, "dev/test1")
-	walker.Files = append(walker.Files, "dev/test1/utils")
-	walker.Files = append(walker.Files, "dev/test2")
-	walker.Files = append(walker.Files, "dev/project")
-	walker.Files = append(walker.Files, "testtt")
-
-	dg, _, err := LoadDiggerConfig(tempDir, walker)
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
-	assert.Equal(t, "test1", dg.Projects[0].Name)
+	assert.Equal(t, "utils", dg.Projects[0].Name)
 	assert.Equal(t, "test2", dg.Projects[1].Name)
-	assert.Equal(t, "dev/test1", dg.Projects[0].Dir)
+	assert.Equal(t, "dev/test1/utils", dg.Projects[0].Dir)
 	assert.Equal(t, "dev/test2", dg.Projects[1].Dir)
 	assert.Equal(t, 2, len(dg.Projects))
 }
@@ -335,16 +353,20 @@ generate_projects:
 `
 	deleteFile := createFile(path.Join(tempDir, "digger.yml"), diggerCfg)
 	defer deleteFile()
-
-	walker := &MockDirWalker{}
-	walker.Files = append(walker.Files, "dev")
-	walker.Files = append(walker.Files, "dev/test1")
-	walker.Files = append(walker.Files, "dev/test1/utils")
-	walker.Files = append(walker.Files, "dev/test2")
-	walker.Files = append(walker.Files, "dev/project")
-	walker.Files = append(walker.Files, "testtt")
-
-	dg, _, err := LoadDiggerConfig(tempDir, walker)
+	dirsToCreate := []string{
+		"dev",
+		"dev/test1",
+		"dev/test1/utils",
+		"dev/test2",
+		"dev/project",
+		"testtt",
+	}
+	for _, dir := range dirsToCreate {
+		err := os.MkdirAll(path.Join(tempDir, dir), os.ModePerm)
+		defer createFile(path.Join(tempDir, dir, "main.tf"), "")()
+		assert.NoError(t, err, "expected error to be nil")
+	}
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
 	assert.Equal(t, "dev", dg.Projects[0].Name)
@@ -359,9 +381,7 @@ func TestMissingProjectsReturnsError(t *testing.T) {
 `
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
-	walker := &MockDirWalker{}
-
-	_, _, err := LoadDiggerConfig(tempDir, walker)
+	_, _, err := LoadDiggerConfig(tempDir)
 	assert.ErrorContains(t, err, "no projects configuration found")
 }
 
@@ -384,7 +404,7 @@ workflows:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	dg, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	dg, _, err := LoadDiggerConfig(tempDir)
 	assert.NoError(t, err, "expected error to be nil")
 	assert.NotNil(t, dg, "expected digger config to be not nil")
 	assert.Equal(t, "my_custom_workflow", dg.Projects[0].Workflow)
@@ -406,7 +426,7 @@ projects:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	_, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	_, _, err := LoadDiggerConfig(tempDir)
 	assert.Error(t, err, "failed to find workflow config 'my_custom_workflow' for project 'my-first-app'")
 
 	// steps block is missing for workflows
@@ -421,7 +441,7 @@ workflows:
 	deleteFile = createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	diggerConfig, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	diggerConfig, _, err := LoadDiggerConfig(tempDir)
 	assert.Equal(t, "my_custom_workflow", diggerConfig.Projects[0].Workflow)
 	workflow, ok := diggerConfig.Workflows["my_custom_workflow"]
 	assert.True(t, ok)
@@ -450,7 +470,7 @@ workflows:
 	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
 	defer deleteFile()
 
-	_, _, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{})
+	_, _, err := LoadDiggerConfig(tempDir)
 	assert.Equal(t, "failed to find workflow config 'my_custom_workflow' for project 'my-first-app'", err.Error())
 
 }
diff --git a/pkg/integration/integration_test.go b/pkg/integration/integration_test.go
index 2f9ac522..53472fc0 100644
--- a/pkg/integration/integration_test.go
+++ b/pkg/integration/integration_test.go
@@ -358,7 +358,7 @@ func TestHappyPath(t *testing.T) {
 	terraform.CreateValidTerraformTestFile(dir)
 	terraform.CreateSingleEnvDiggerYmlFile(dir)
 
-	diggerConfig, depGraph, err := configuration.LoadDiggerConfig(dir, &configuration.FileSystemDirWalker{})
+	diggerConfig, depGraph, err := configuration.LoadDiggerConfig(dir)
 	assert.NoError(t, err)
 
 	lock, err := locking.GetLock()
@@ -510,7 +510,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
 	terraform.CreateValidTerraformTestFile(dir)
 	terraform.CreateMultiEnvDiggerYmlFile(dir)
 
-	diggerConfig, depGraph, err := configuration.LoadDiggerConfig(dir, &configuration.FileSystemDirWalker{})
+	diggerConfig, depGraph, err := configuration.LoadDiggerConfig(dir)
 	assert.NoError(t, err)
 
 	sess, err := session.NewSessionWithOptions(session.Options{
@@ -733,7 +733,7 @@ workflows:
 	terraform.CreateValidTerraformTestFile(dir)
 	terraform.CreateCustomDiggerYmlFile(dir, diggerCfg)
 
-	diggerConfig, depGraph, err := configuration.LoadDiggerConfig(dir, &configuration.FileSystemDirWalker{})
+	diggerConfig, depGraph, err := configuration.LoadDiggerConfig(dir)
 	assert.NoError(t, err)
 
 	assert.NotNil(t, diggerConfig.Workflows)