Repository: prometheus/prometheus (language: go)

Issue:
promtool return zero code on validation error (unmarshall errors)
### What did you do?

```
$ promtool check config inputs/prometheus/prometheus.yml; echo $?
  FAILED: parsing YAML file /tmp/prometheus.yml: yaml: unmarshal errors:
  line 22: field relabel_configs not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }
  line 23: field source_labels not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }
  line 24: field target_label not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }
  line 25: field source_labels not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }
  line 26: field target_label not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }
  line 27: field target_label not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }
  line 28: field replacement not found in type struct { Targets []string "yaml:\"targets\""; Labels model.LabelSet "yaml:\"labels\"" }

0
```


### What did you expect to see?

Promtool fails with non-zero return code

### What did you see instead? Under which circumstances?

Zero return code

### System information

Linux 5.17.7-300.fc36.x86_64 x86_64

### Prometheus version

```text
promtool, version 2.36.2 (branch: HEAD, revision: d7e7b8e04b5ecdc1dd153534ba376a622b72741b)
  build user:       root@f051ce0d6050
  build date:       20220620-13:21:35
  go version:       go1.18.3
  platform:         linux/amd64
```


### Prometheus configuration file

```yaml
global:
  scrape_interval:     20s
  evaluation_interval: 5s

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
      - targets:
        - alertmanager:9093

# Scrape configurations
scrape_configs:
  - job_name: "invalid"
    metrics_path: /probe
    params:
      module:
        - http_fuse
    static_configs:
      - targets:
        - https://someurl
        relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115
```


### Alertmanager version

_No response_

### Alertmanager configuration file

_No response_

### Logs

_No response_


Discussion / hints:
Hello, downgrading Prometheus to 2.36.0 solved this problem for me.
based on this PR https://github.com/prometheus/prometheus/pull/10840, I think you should enable the flag `--lint-fatal` and then it should rework as before.

Fix patch:
diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go
index ad802e48240..96d7f3851a9 100644
--- a/cmd/promtool/main.go
+++ b/cmd/promtool/main.go
@@ -344,6 +344,7 @@ func CheckConfig(agentMode, checkSyntaxOnly bool, lintSettings lintConfig, files
 		ruleFiles, err := checkConfig(agentMode, f, checkSyntaxOnly)
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:", err)
+			hasErrors = true
 			failed = true
 		} else {
 			if len(ruleFiles) > 0 {


Test patch:
diff --git a/cmd/promtool/main_test.go b/cmd/promtool/main_test.go
index d4773fc918b..40df3e5248e 100644
--- a/cmd/promtool/main_test.go
+++ b/cmd/promtool/main_test.go
@@ -14,13 +14,16 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"os/exec"
 	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -30,6 +33,21 @@ import (
 	"github.com/prometheus/prometheus/model/rulefmt"
 )
 
+var promtoolPath = os.Args[0]
+
+func TestMain(m *testing.M) {
+	for i, arg := range os.Args {
+		if arg == "-test.main" {
+			os.Args = append(os.Args[:i], os.Args[i+1:]...)
+			main()
+			return
+		}
+	}
+
+	exitCode := m.Run()
+	os.Exit(exitCode)
+}
+
 func TestQueryRange(t *testing.T) {
 	s, getRequest := mockServer(200, `{"status": "success", "data": {"resultType": "matrix", "result": []}}`)
 	defer s.Close()
@@ -359,3 +377,59 @@ func TestCheckMetricsExtended(t *testing.T) {
 		},
 	}, stats)
 }
+
+func TestExitCodes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	for _, c := range []struct {
+		file      string
+		exitCode  int
+		lintIssue bool
+	}{
+		{
+			file: "prometheus-config.good.yml",
+		},
+		{
+			file:     "prometheus-config.bad.yml",
+			exitCode: 1,
+		},
+		{
+			file:     "prometheus-config.nonexistent.yml",
+			exitCode: 1,
+		},
+		{
+			file:      "prometheus-config.lint.yml",
+			lintIssue: true,
+			exitCode:  3,
+		},
+	} {
+		t.Run(c.file, func(t *testing.T) {
+			for _, lintFatal := range []bool{true, false} {
+				t.Run(fmt.Sprintf("%t", lintFatal), func(t *testing.T) {
+					args := []string{"-test.main", "check", "config", "testdata/" + c.file}
+					if lintFatal {
+						args = append(args, "--lint-fatal")
+					}
+					tool := exec.Command(promtoolPath, args...)
+					err := tool.Run()
+					if c.exitCode == 0 || (c.lintIssue && !lintFatal) {
+						require.NoError(t, err)
+						return
+					}
+
+					require.Error(t, err)
+
+					var exitError *exec.ExitError
+					if errors.As(err, &exitError) {
+						status := exitError.Sys().(syscall.WaitStatus)
+						require.Equal(t, c.exitCode, status.ExitStatus())
+					} else {
+						t.Errorf("unable to retrieve the exit status for promtool: %v", err)
+					}
+				})
+			}
+		})
+	}
+}
diff --git a/cmd/promtool/testdata/prometheus-config.bad.yml b/cmd/promtool/testdata/prometheus-config.bad.yml
new file mode 100644
index 00000000000..13e262e0519
--- /dev/null
+++ b/cmd/promtool/testdata/prometheus-config.bad.yml
@@ -0,0 +1,1 @@
+not-prometheus:
diff --git a/cmd/promtool/testdata/prometheus-config.good.yml b/cmd/promtool/testdata/prometheus-config.good.yml
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/cmd/promtool/testdata/prometheus-config.lint.yml b/cmd/promtool/testdata/prometheus-config.lint.yml
new file mode 100644
index 00000000000..ffa7c38434e
--- /dev/null
+++ b/cmd/promtool/testdata/prometheus-config.lint.yml
@@ -0,0 +1,2 @@
+rule_files:
+  - prometheus-rules.lint.yml
diff --git a/cmd/promtool/testdata/prometheus-rules.lint.yml b/cmd/promtool/testdata/prometheus-rules.lint.yml
new file mode 100644
index 00000000000..b067675de1a
--- /dev/null
+++ b/cmd/promtool/testdata/prometheus-rules.lint.yml
@@ -0,0 +1,17 @@
+groups:
+- name: example
+  rules:
+  - alert: HighRequestLatency
+    expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5
+    for: 10m
+    labels:
+      severity: page
+    annotations:
+      summary: High request latency
+  - alert: HighRequestLatency
+    expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5
+    for: 10m
+    labels:
+      severity: page
+    annotations:
+      summary: High request latency