Repository: plexsystems/konstraint
Pull request #38 (2020-07-31 02:46:34)

Support recursive imports
Adds support for libraries importing other libraries even if the policy does not import them. Addresses #30

jpreese: I know you have this marked as draft @jalseth but I think this looks great. Simple and straight forward. This will be huge, thanks!

Patch:
diff --git a/README.md b/README.md
index 73af204..31fdac7 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,3 @@ To create the Gatekeeper resources, use `konstraint create <policy_dir>`. To gen
 **Konstraint ran without error, but I don't see any new files.**
 
 This typically means no policies were found, or the policies did not have any `violation[]` rules so they are not compatible with Gatekeeper. For more information, see [How Constraints are Created](docs/constraint_creation.md).
-
-**I imported `lib.XYZ` but Gatekeeper cannot locate `lib.core`?**
-
-At this time, Konstraint only parses the imports from the policy. As many of the libraries rely on `lib.core`, if this is not imported in your policy it will not be included in the generated `ConstraintTemplate` which means not all of the necessary libraries have been included. This issue is being tracked in [https://github.com/plexsystems/konstraint/issues/30](https://github.com/plexsystems/konstraint/issues/30).
diff --git a/docs/cli/konstraint.md b/docs/cli/konstraint.md
index 652ed7d..47097fd 100644
--- a/docs/cli/konstraint.md
+++ b/docs/cli/konstraint.md
@@ -17,4 +17,4 @@ A tool to create and manage Gatekeeper CRDs from Rego
 * [konstraint create](konstraint_create.md)	 - Create Gatekeeper constraints from Rego policies
 * [konstraint doc](konstraint_doc.md)	 - Generate documentation from Rego policies
 
-###### Auto generated by spf13/cobra on 28-Jul-2020
+###### Auto generated by spf13/cobra on 31-Jul-2020
diff --git a/docs/cli/konstraint_create.md b/docs/cli/konstraint_create.md
index d2093c4..ab1a7ff 100644
--- a/docs/cli/konstraint_create.md
+++ b/docs/cli/konstraint_create.md
@@ -35,4 +35,4 @@ Create constraints with the Gatekeeper enforcement action set to dryrun
 
 * [konstraint](konstraint.md)	 - Konstraint
 
-###### Auto generated by spf13/cobra on 28-Jul-2020
+###### Auto generated by spf13/cobra on 31-Jul-2020
diff --git a/docs/cli/konstraint_doc.md b/docs/cli/konstraint_doc.md
index c240058..7495b8d 100644
--- a/docs/cli/konstraint_doc.md
+++ b/docs/cli/konstraint_doc.md
@@ -31,4 +31,4 @@ Save the documentation to a specific directory
 
 * [konstraint](konstraint.md)	 - Konstraint
 
-###### Auto generated by spf13/cobra on 28-Jul-2020
+###### Auto generated by spf13/cobra on 31-Jul-2020
diff --git a/examples/deny-privileged-containers/template.yaml b/examples/deny-privileged-containers/template.yaml
deleted file mode 100755
index 82e9384..0000000
--- a/examples/deny-privileged-containers/template.yaml
+++ /dev/null
@@ -1,171 +0,0 @@
-apiVersion: templates.gatekeeper.sh/v1beta1
-kind: ConstraintTemplate
-metadata:
-  creationTimestamp: null
-  name: denyprivilegedcontainers
-spec:
-  crd:
-    spec:
-      names:
-        kind: DenyPrivilegedContainers
-  targets:
-  - libs:
-    - |
-      package lib.core
-
-      default is_gatekeeper = false
-
-      is_gatekeeper {
-        has_field(input, "review")
-        has_field(input.review, "object")
-      }
-
-      resource = input {
-        not is_gatekeeper
-      }
-
-      resource = input.review.object {
-        is_gatekeeper
-      }
-
-      apiVersion = resource.apiVersion
-      kind = resource.kind
-      name = resource.metadata.name
-      labels = resource.metadata.labels
-
-      format(msg) = gatekeeper_format {
-        is_gatekeeper
-        gatekeeper_format = {"msg": msg}
-      }
-
-      format(msg) = msg {
-        not is_gatekeeper
-      }
-
-      has_field(obj, field) {
-        obj[field]
-      }
-
-      missing_field(obj, field) = true {
-        obj[field] == ""
-      }
-
-      missing_field(obj, field) = true {
-        not has_field(obj, field)
-      }
-    - |
-      package lib.workloads
-
-      import data.lib.core
-
-      is_statefulset {
-        lower(core.kind) == "statefulset"
-      }
-
-      is_daemonset {
-        lower(core.kind) == "daemonset"
-      }
-
-      is_deployment {
-        lower(core.kind) == "deployment"
-      }
-
-      is_job {
-        lower(core.kind) == "job"
-      }
-
-      is_pod {
-        lower(core.kind) == "pod"
-      }
-
-      is_workload {
-        containers[_]
-      }
-
-      pods[pod] {
-        is_statefulset
-        pod = core.resource.spec.template
-      }
-
-      pods[pod] {
-        is_daemonset
-        pod = core.resource.spec.template
-      }
-
-      pods[pod] {
-        is_deployment
-        pod = core.resource.spec.template
-      }
-
-      pods[pod] {
-        is_job
-        pod = core.resource.spec.template
-      }
-
-      pods[pod] {
-        is_pod
-        pod = core.resource
-      }
-
-      volumes[volume] {
-        pods[pod]
-        volume = pod.spec.volumes[_]
-      }
-
-      pod_containers(pod) = all_containers {
-        keys = {"containers", "initContainers"}
-        all_containers = [c | keys[k]; c = pod.spec[k][_]]
-      }
-
-      containers[container] {
-        pods[pod]
-        all_containers = pod_containers(pod)
-        container = all_containers[_]
-      }
-
-      containers[container] {
-        all_containers = pod_containers(core.object)
-        container = all_containers[_]
-      }
-    - |
-      package lib.security
-
-      dropped_capability(container, cap) {
-        container.securityContext.capabilities.drop[_] == cap
-      }
-
-      added_capability(container, cap) {
-        container.securityContext.capabilities.add[_] == cap
-      }
-
-      dropped_capability(psp, cap) {
-        psp.spec.capabilities.drop[_] == cap
-      }
-
-      added_capability(psp, cap) {
-        psp.spec.capabilities.add[_] == cap
-      }
-    rego: |
-      package deny_privileged_containers
-
-      import data.lib.core
-      import data.lib.workloads
-      import data.lib.security
-
-
-      violation[msg] {
-        workloads.containers[container]
-        is_privileged(container)
-
-        msg = core.format(sprintf("(%s) %s: Containers are not allowed to run as privileged", [core.kind, core.name]))
-      }
-
-      is_privileged(container) {
-        container.securityContext.privileged
-      }
-
-      is_privileged(container) {
-        security.added_capability(container, "CAP_SYS_ADMIN")
-      }
-    target: admission.k8s.gatekeeper.sh
-status: {}
diff --git a/internal/commands/create.go b/internal/commands/create.go
index b503181..434e36c 100644
--- a/internal/commands/create.go
+++ b/internal/commands/create.go
@@ -94,9 +94,10 @@ func runCreateCommand(path string) error {
 			constraintFileName = fmt.Sprintf("constraint_%s.yaml", GetKindFromPath(policy.FilePath))
 		}
 
-		matchingLibraries := getMatchingLibraries(policy, libraries)
-		if len(matchingLibraries) != len(policy.ImportPackages) {
-			return fmt.Errorf("missing imported libraries")
+		importedLibraries := getImportedLibraries(policy, libraries)
+		var librariesContents []string
+		for _, library := range importedLibraries {
+			librariesContents = append(librariesContents, getRegoWithoutComments(library.Contents))
 		}
 
 		if _, err := os.Stat(outputDir); os.IsNotExist(err) {
@@ -106,7 +107,7 @@ func runCreateCommand(path string) error {
 			}
 		}
 
-		constraintTemplate := getConstraintTemplate(policy, matchingLibraries)
+		constraintTemplate := getConstraintTemplate(policy, librariesContents)
 		constraintTemplateBytes, err := yaml.Marshal(&constraintTemplate)
 		if err != nil {
 			return fmt.Errorf("marshal constrainttemplate: %w", err)
@@ -248,15 +249,39 @@ func getLibraryPath(path string) (string, error) {
 	return libraryPath, nil
 }
 
-func getMatchingLibraries(policy rego.File, libraries []rego.File) []string {
-	var libs []string
-	for _, importPackage := range policy.ImportPackages {
-		for _, library := range libraries {
-			if importPackage == library.PackageName {
-				libs = append(libs, library.Contents)
+func getImportedLibraries(file rego.File, allLibraries []rego.File) []rego.File {
+	var importedLibraries []rego.File
+	for _, importedPackage := range file.ImportPackages {
+		for _, library := range allLibraries {
+			if library.PackageName == importedPackage {
+				importedLibraries = append(importedLibraries, library)
 			}
 		}
 	}
 
-	return libs
+	for _, library := range importedLibraries {
+		importedLibraries = append(importedLibraries, getImportedLibraries(library, allLibraries)...)
+	}
+
+	importedLibraries = getUniqueRegoFiles(importedLibraries)
+	return importedLibraries
+}
+
+func getUniqueRegoFiles(files []rego.File) []rego.File {
+	var uniqueFiles []rego.File
+	for _, file := range files {
+		var seen bool
+		for _, uniqueFile := range uniqueFiles {
+			if file.PackageName == uniqueFile.PackageName {
+				seen = true
+				break
+			}
+		}
+
+		if !seen {
+			uniqueFiles = append(uniqueFiles, file)
+		}
+	}
+
+	return uniqueFiles
 }
diff --git a/internal/commands/document.go b/internal/commands/document.go
index 0bef0b3..407f0ce 100644
--- a/internal/commands/document.go
+++ b/internal/commands/document.go
@@ -181,6 +181,10 @@ func getDocumentation(path string, severity string, outputDirectory string) ([]D
 		}
 		relDir := filepath.Dir(relPath)
 
+		// Markdown specification notes that all pathing should be represented
+		// with a forward slash.
+		relDir = strings.ReplaceAll(relDir, string(os.PathSeparator), "/")
+
 		regoWithoutComments := getRegoWithoutComments(policy.Contents)
 
 		document := Document{
diff --git a/internal/rego/rego.go b/internal/rego/rego.go
index bcaeaae..9dc4993 100644
--- a/internal/rego/rego.go
+++ b/internal/rego/rego.go
@@ -185,6 +185,9 @@ func readFilesContents(filePaths []string) (map[string]string, error) {
 			return nil, fmt.Errorf("read file: %w", err)
 		}
 
+		// Many YAML parsers do not like rendering out CRLF when writing the YAML to disk.
+		// This causes ConstraintTemplates to be rendered with the line breaks as text,
+		// rather than the actual line break.
 		filesContents[filePath] = strings.ReplaceAll(string(data), "\r", "")
 	}
 


Test patch:
diff --git a/internal/commands/create_test.go b/internal/commands/create_test.go
index 5b0861a..83654f7 100644
--- a/internal/commands/create_test.go
+++ b/internal/commands/create_test.go
@@ -90,9 +90,13 @@ rule[msg] { msg = true }`
 		libraries = append(libraries, lib)
 	}
 
-	matchingLibraries := getMatchingLibraries(policyFile, libraries)
+	var librariesContents []string
+	importedLibraries := getImportedLibraries(policyFile, libraries)
+	for _, library := range importedLibraries {
+		librariesContents = append(librariesContents, getRegoWithoutComments(library.Contents))
+	}
 
-	actual := getConstraintTemplate(policyFile, matchingLibraries)
+	actual := getConstraintTemplate(policyFile, librariesContents)
 	if err != nil {
 		t.Fatal("get constraint:", err)
 	}
@@ -107,6 +111,38 @@ rule[msg] { msg = true }`
 	}
 }
 
+func TestRecursiveLibraryImport(t *testing.T) {
+	policyImportsLibA := `package test
+import data.lib.a`
+
+	libFiles := []struct {
+		path     string
+		contents string
+	}{
+		{path: "lib_a.rego", contents: "package lib.a\nimport data.lib.b"},
+		{path: "lib_b.rego", contents: "package lib.b"},
+	}
+
+	policy, err := rego.NewFile("test.rego", policyImportsLibA)
+	if err != nil {
+		t.Fatal("new policy file", err)
+	}
+
+	var libs []rego.File
+	for _, libFile := range libFiles {
+		lib, err := rego.NewFile(libFile.path, libFile.contents)
+		if err != nil {
+			t.Fatal("new library file", err)
+		}
+		libs = append(libs, lib)
+	}
+
+	importedLibraries := getImportedLibraries(policy, libs)
+	if len(importedLibraries) != 2 {
+		t.Error("recursive library import failed")
+	}
+}
+
 func TestGetKindFromPath(t *testing.T) {
 	path := "/path/to/rego/container-resource-limits/something.rego"