Repository: linkml/linkml

Issue:
linkml-lint exit codes
I would prefer for the linter to exit with code 0 if it did not find errors and just emitted warnings.

If others prefer that warnings lead to a non-zero exit code, having exit codes that correspond to warnings only vs errors would be good, e.g. 

warnings => exit 1
errors => exit 10
warnings + errors => exit 11

Alternatively, having a command line option to ignore warnings when assessing how to exit the linter would be good.

Motivation: I run linkml-lint as a GitHub action and was surprised when the linter failed after emitting warnings about non-canonical URIs in the prefixes section.

If these features are already available, can documentation be added so it's obvious for users?

Discussion:

Patch:
diff --git a/docs/schemas/linter.md b/docs/schemas/linter.md
index 1e1759cc..a5fb0336 100644
--- a/docs/schemas/linter.md
+++ b/docs/schemas/linter.md
@@ -126,3 +126,11 @@ For example, to generate a markdown report in a file named `linter-results.md`:
 ```bash
 linkml-lint --format markdown --output linter-results.md myschema.yaml
 ```
+
+## Exit Codes
+
+If the linter does not encounter any rule violations at all it will exit with code `0`. 
+
+If the linter encounters rule violations with `level: error` it will exit with code `2`. This will be the case regardless of whether there are also rule violations with `level: warning`.
+
+By default, if the linter encounters _only_ rule violations with `level: warning` it will exit with code `1`. This behavior can be changed with command line options. In this scenario, if the `--ignore-warnings` flag is provided the exit code will be `0`. If instead the `--max-warnings <int>` option is passed, the exit code will be `1` or `0` depending on whether the number of warning rule violations is greater than the provided number or not. If both `--ignore-warnings` and `--max-warnings` are used `--ignore-warnings` takes precedence. 
diff --git a/linkml/linter/cli.py b/linkml/linter/cli.py
index bc99323b..07483e43 100644
--- a/linkml/linter/cli.py
+++ b/linkml/linter/cli.py
@@ -6,6 +6,7 @@ from typing import Iterable
 import click
 import yaml
 
+from .config.datamodel.config import RuleLevel
 from .formatters import (JsonFormatter, MarkdownFormatter, TerminalFormatter,
                          TsvFormatter)
 from .linter import Linter
@@ -37,17 +38,49 @@ def get_yaml_files(root: Path) -> Iterable[str]:
     ),
 )
 @click.option(
-    "-c", "--config", type=click.Path(exists=True, dir_okay=False, resolve_path=True)
+    "-c",
+    "--config",
+    type=click.Path(exists=True, dir_okay=False, resolve_path=True),
+    help="Custom linter configuration file.",
 )
 @click.option(
     "-f",
     "--format",
     type=click.Choice(["terminal", "markdown", "json", "tsv"]),
     default="terminal",
+    help="Report format.",
+    show_default=True,
+)
+@click.option(
+    "-o", "--output", type=click.File("w"), default="-", help="Report file name."
+)
+@click.option(
+    "--ignore-warnings",
+    is_flag=True,
+    default=False,
+    help="Do not exit with an error status if only warnings are found.",
+)
+@click.option(
+    "--max-warnings",
+    type=int,
+    default=0,
+    show_default=True,
+    help="Do not exit with an error status if up to this number of warnings (and no errors) are found.",
 )
-@click.option("-o", "--output", type=click.File("w"), default="-")
 @click.option("--fix/--no-fix", default=False)
-def main(schema: Path, fix: bool, config: str, format: str, output):
+def main(
+    schema: Path,
+    fix: bool,
+    config: str,
+    format: str,
+    output,
+    ignore_warnings: bool,
+    max_warnings: int,
+):
+    """Run linter on SCHEMA.
+
+    SCHEMA can be a single LinkML YAML file or a directory. If it is a directory
+    every YAML file found in the directory (recursively) will be linted."""
     config_file = None
     if config:
         config_file = config
@@ -74,17 +107,27 @@ def main(schema: Path, fix: bool, config: str, format: str, output):
     elif format == "tsv":
         formatter = TsvFormatter(output)
 
-    exit_code = 0
+    error_count = 0
+    warning_count = 0
     formatter.start_report()
     for path in get_yaml_files(schema):
         formatter.start_schema(path)
         report = linter.lint(path, fix=fix)
         for problem in report:
-            exit_code = 1
+            if str(problem.level) is RuleLevel.error.text:
+                error_count += 1
+            elif str(problem.level) is RuleLevel.warning.text:
+                warning_count += 1
             formatter.handle_problem(problem)
         formatter.end_schema()
     formatter.end_report()
 
+    exit_code = 0
+    if error_count > 0:
+        exit_code = 2
+    elif not ignore_warnings and warning_count > max_warnings:
+        exit_code = 1
+
     sys.exit(exit_code)
 
 


Test patch:
diff --git a/tests/test_linter/test_cli.py b/tests/test_linter/test_cli.py
index ee4cb89d..a35a12d4 100644
--- a/tests/test_linter/test_cli.py
+++ b/tests/test_linter/test_cli.py
@@ -36,7 +36,9 @@ classes:
         )
 
 
-def write_config_file(name: str, extends_recommended: bool = False) -> None:
+def write_config_file(
+    name: str, extends_recommended: bool = False, tree_root_level: str = "error"
+) -> None:
     with open(name, "w") as f:
         if extends_recommended:
             f.write(
@@ -45,10 +47,10 @@ extends: recommended
 """
             )
         f.write(
-            """
+            f"""
 rules:
   tree_root_class:
-    level: error
+    level: {tree_root_level}
 """
         )
 
@@ -62,7 +64,7 @@ class TestLinterCli(unittest.TestCase):
             write_schema_file()
 
             result = self.runner.invoke(main, [SCHEMA_FILE])
-            self.assertEqual(result.exit_code, 1)
+            self.assertEqual(result.exit_code, 2)
             self.assertIn(
                 "warning  class_definition 'Adult' does not have recommended slot 'description'  (recommended)",
                 result.stdout,
@@ -81,7 +83,7 @@ class TestLinterCli(unittest.TestCase):
             write_config_file(".linkmllint.yaml")
 
             result = self.runner.invoke(main, [SCHEMA_FILE])
-            self.assertEqual(result.exit_code, 1)
+            self.assertEqual(result.exit_code, 2)
             self.assertIn(
                 "error    Schema does not have class with `tree_root: true`  (tree_root_class)",
                 result.stdout,
@@ -95,7 +97,7 @@ class TestLinterCli(unittest.TestCase):
             write_config_file(config_file)
 
             result = self.runner.invoke(main, ["--config", config_file, SCHEMA_FILE])
-            self.assertEqual(result.exit_code, 1)
+            self.assertEqual(result.exit_code, 2)
             self.assertIn(
                 "error    Schema does not have class with `tree_root: true`  (tree_root_class)",
                 result.stdout,
@@ -109,7 +111,7 @@ class TestLinterCli(unittest.TestCase):
             write_config_file(config_file, extends_recommended=True)
 
             result = self.runner.invoke(main, ["--config", config_file, SCHEMA_FILE])
-            self.assertEqual(result.exit_code, 1)
+            self.assertEqual(result.exit_code, 2)
             self.assertIn(
                 "error    Schema does not have class with `tree_root: true`  (tree_root_class)",
                 result.stdout,
@@ -118,6 +120,72 @@ class TestLinterCli(unittest.TestCase):
                 "warning  Class has name 'person'  (standard_naming)", result.stdout
             )
 
+    def test_warning_exit_code(self):
+        config_file = "config.yaml"
+        with self.runner.isolated_filesystem():
+            write_schema_file()
+            write_config_file(
+                config_file, extends_recommended=False, tree_root_level="warning"
+            )
+
+            result = self.runner.invoke(main, ["--config", config_file, SCHEMA_FILE])
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(
+                "warning  Schema does not have class with `tree_root: true`  (tree_root_class)",
+                result.stdout,
+            )
+
+    def test_ignore_warnings_flag(self):
+        config_file = "config.yaml"
+        with self.runner.isolated_filesystem():
+            write_schema_file()
+            write_config_file(
+                config_file, extends_recommended=False, tree_root_level="warning"
+            )
+
+            result = self.runner.invoke(
+                main, ["--config", config_file, "--ignore-warnings", SCHEMA_FILE]
+            )
+            self.assertEqual(result.exit_code, 0)
+            self.assertIn(
+                "warning  Schema does not have class with `tree_root: true`  (tree_root_class)",
+                result.stdout,
+            )
+
+    def test_max_warnings_flag(self):
+        config_file = "config.yaml"
+        with self.runner.isolated_filesystem():
+            write_schema_file()
+            write_config_file(
+                config_file, extends_recommended=False, tree_root_level="warning"
+            )
+
+            result = self.runner.invoke(
+                main, ["--config", config_file, "--max-warnings", 1, SCHEMA_FILE]
+            )
+            self.assertEqual(result.exit_code, 0)
+            self.assertIn(
+                "warning  Schema does not have class with `tree_root: true`  (tree_root_class)",
+                result.stdout,
+            )
+
+    def test_exceeded_max_warnings_flag(self):
+        config_file = "config.yaml"
+        with self.runner.isolated_filesystem():
+            write_schema_file()
+            write_config_file(
+                config_file, extends_recommended=False, tree_root_level="warning"
+            )
+
+            result = self.runner.invoke(
+                main, ["--config", config_file, "--max-warnings", 0, SCHEMA_FILE]
+            )
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(
+                "warning  Schema does not have class with `tree_root: true`  (tree_root_class)",
+                result.stdout,
+            )
+
     def test_no_schema_errors(self):
         with self.runner.isolated_filesystem():
             with open(SCHEMA_FILE, "w") as f: