Repository: oliver006/redis_exporter

Issue:
monitor aof incr file size
**Describe the problem**
Due to the way AOF persistence works, sometimes the `AOFREWRITE` command must be sent to the Redis server manually. Even then, the AOF incr file can grow much before the rewrite is triggered.

It'd be interesting to add a new map metric for the incr file size, which can later trigger alerts when surpassing a threshold. 

The `INFO` command exports the following AOF-related metrics:
- aof_last_cow_size
- aof_current_size
- aof_base_size
- aof_pending_rewrite
- aof_buffer_length
- aof_rewrite_buffer_length

However, none of them exports directly the size for the incr files, only the total sum of the base RDB + all incr files.

The proposal is to add a new metric `aof_incr_file_size` with the label `filename` where it will constantly monitor all AOF incr files from the path retrieved from the redis config:
Base Dir: `CONFIG GET dir`
Appendonly Dir: `CONFIG GET appenddirname`

The limitation is that it will only work if `redis_exporter` has access to the same directories as the redis server.

**What version of redis_exporter are you running?**
Please run `redis_exporter --version` if you're not sure what version you're running.
- [x] v1.67.0 


Hints from issue comments:
@oliver006: Closing as per 
https://github.com/oliver006/redis_exporter/pull/978#issuecomment-2672610924

Solution patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 92f05c5..01797fa 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -26,6 +26,9 @@ jobs:
           make test-certs
           chmod 777 ./contrib/tls/*
 
+      - name: Create data folder
+        run: mkdir -p ./data
+
       - name: Start services
         run: docker-compose up -d
         working-directory: ./
diff --git a/Makefile b/Makefile
index 369a080..9e74419 100644
--- a/Makefile
+++ b/Makefile
@@ -53,6 +53,7 @@ test:
 	TEST_TILE38_URI="redis://localhost:19851" \
 	TEST_REDIS_SENTINEL_URI="redis://localhost:26379" \
 	TEST_REDIS_MODULES_URI="redis://localhost:36379" \
+	TEST_AOF_FILE_SIZE_URL="redis://localhost:16501" \
 	go test -v -covermode=atomic -cover -race -coverprofile=coverage.txt -p 1 ./...
 
 .PHONY: lint
diff --git a/docker-compose.yml b/docker-compose.yml
index d06bfde..5dc6da2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -121,6 +121,15 @@ services:
     ports:
       - "19851:9851"
 
+  redis-aof:
+    image: redis:7.4
+    command: "redis-server  --enable-debug-command yes --protected-mode no --appendonly yes"
+    ports:
+      - "16501:6379"
+      - "6379:6379"
+    volumes:
+      - ./data:/data
+
   redis-stack:
     image: redis/redis-stack-server:7.4.0-v0
     ports:
diff --git a/exporter/aof.go b/exporter/aof.go
new file mode 100644
index 0000000..c375ed4
--- /dev/null
+++ b/exporter/aof.go
@@ -0,0 +1,93 @@
+package exporter
+
+import (
+	"os"
+	"strings"
+
+	"github.com/gomodule/redigo/redis"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+)
+
+func getAofFilePath(c redis.Conn, configCommandName string) (string, error) {
+
+	dir_response, err := redis.StringMap(doRedisCmd(c, configCommandName, "GET", "dir"))
+	if err != nil {
+		return "", err
+	}
+	dir := dir_response["dir"]
+	appenddirname_response, err := redis.StringMap(doRedisCmd(c, "CONFIG", "GET", "appenddirname"))
+	appenddirname := appenddirname_response["appenddirname"]
+	if err != nil {
+		return "", err
+	}
+
+	return dir + "/" + appenddirname, nil
+}
+
+func listAofIncrFiles(dir string) ([]string, error) {
+	// List all aof incr files from appendonly dir
+
+	files, err := os.ReadDir(dir)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var incrFiles []string
+	for _, file := range files {
+		if strings.Contains(file.Name(), "incr.aof") {
+			incrFiles = append(incrFiles, file.Name())
+		}
+	}
+
+	return incrFiles, nil
+}
+
+func getFileSize(dir string, file string) (float64, error) {
+
+	info, err := os.Stat(dir + string(os.PathSeparator) + file)
+	if err != nil {
+		return 0, err
+	}
+
+	return float64(info.Size()), nil
+
+}
+
+func (e *Exporter) extractAofFileSizeMetrics(ch chan<- prometheus.Metric, c redis.Conn, configCommandName string, overrideAofFilePath string) {
+
+	log.Debug("extractAofFileSizeMetrics()")
+
+	var dir string
+	var err error
+
+	if overrideAofFilePath != "" {
+		dir = overrideAofFilePath
+	} else {
+		dir, err = getAofFilePath(c, configCommandName)
+		if err != nil {
+			log.Errorf("extractAofFileSizeMetrics() err: %s", err)
+			return
+		}
+	}
+
+	incrFiles, err := listAofIncrFiles(dir)
+	log.Debug("incrFiles: ", incrFiles)
+	if err != nil {
+		log.Errorf("extractAofFileSizeMetrics() err: %s", err)
+		return
+	}
+
+	for _, file := range incrFiles {
+		size, err := getFileSize(dir, file)
+		log.Debug("file: ", file, " size: ", size)
+		if err != nil {
+			log.Errorf("extractAofFileSizeMetrics() err: %s", err)
+			continue
+		}
+		filename := strings.ReplaceAll(file, ".", "_")
+		e.registerConstMetricGauge(ch, "aof_file_size_bytes", size, filename)
+	}
+
+}
diff --git a/exporter/exporter.go b/exporter/exporter.go
index 30cc227..f69be8b 100644
--- a/exporter/exporter.go
+++ b/exporter/exporter.go
@@ -76,6 +76,8 @@ type Options struct {
 	IsCluster                      bool
 	ExportClientList               bool
 	ExportClientsInclPort          bool
+	InclAofFileSize                bool
+	OverrideAofFilePath            string
 	ConnectionTimeouts             time.Duration
 	MetricsPath                    string
 	RedisMetricsOnly               bool
@@ -238,6 +240,7 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) {
 			"aof_delayed_fsync":            "aof_delayed_fsync",
 			"aof_last_bgrewrite_status":    "aof_last_bgrewrite_status",
 			"aof_last_write_status":        "aof_last_write_status",
+			"aof_incr_file_size":           "aof_incr_file_size_bytes",
 			"module_fork_in_progress":      "module_fork_in_progress",
 			"module_fork_last_cow_size":    "module_fork_last_cow_size",
 
@@ -481,6 +484,7 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) {
 		"stream_radix_tree_nodes":                            {txt: `Radix tree nodes count`, lbls: []string{"db", "stream"}},
 		"up":                                                 {txt: "Information about the Redis instance"},
 		"module_info":                                        {txt: "Information about loaded Redis module", lbls: []string{"name", "ver", "api", "filters", "usedby", "using"}},
+		"aof_file_size_bytes":                                {txt: "AOF file size in bytes", lbls: []string{"filename"}},
 	} {
 		e.metricDescriptions[k] = newMetricDescr(opts.Namespace, k, desc.txt, desc.lbls)
 	}
@@ -762,6 +766,10 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
 		e.extractModulesMetrics(ch, c)
 	}
 
+	if e.options.InclAofFileSize {
+		e.extractAofFileSizeMetrics(ch, c, e.options.ConfigCommandName, e.options.OverrideAofFilePath)
+	}
+
 	if len(e.options.LuaScript) > 0 {
 		for filename, script := range e.options.LuaScript {
 			if err := e.extractLuaScriptMetrics(ch, c, filename, script); err != nil {
diff --git a/main.go b/main.go
index 1eac5f4..cbf1a60 100644
--- a/main.go
+++ b/main.go
@@ -95,6 +95,8 @@ func main() {
 		pingOnConnect                  = flag.Bool("ping-on-connect", getEnvBool("REDIS_EXPORTER_PING_ON_CONNECT", false), "Whether to ping the redis instance after connecting")
 		inclConfigMetrics              = flag.Bool("include-config-metrics", getEnvBool("REDIS_EXPORTER_INCL_CONFIG_METRICS", false), "Whether to include all config settings as metrics")
 		inclModulesMetrics             = flag.Bool("include-modules-metrics", getEnvBool("REDIS_EXPORTER_INCL_MODULES_METRICS", false), "Whether to collect Redis Modules metrics")
+		inclAofFileSize                = flag.Bool("include-aof-file-size", getEnvBool("REDIS_EXPORTER_INCL_AOF_FILE_SIZE", false), "Whether to collect AOF file size metrics")
+		overrideAofFilePath            = flag.String("override-aof-file-path", getEnv("REDIS_EXPORTER_OVERRIDE_AOF_FILE_PATH", ""), "Override the AOF file path in the metrics")
 		disableExportingKeyValues      = flag.Bool("disable-exporting-key-values", getEnvBool("REDIS_EXPORTER_DISABLE_EXPORTING_KEY_VALUES", false), "Whether to disable values of keys stored in redis as labels or not when using check-keys/check-single-key")
 		excludeLatencyHistogramMetrics = flag.Bool("exclude-latency-histogram-metrics", getEnvBool("REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS", false), "Do not try to collect latency histogram metrics")
 		redactConfigMetrics            = flag.Bool("redact-config-metrics", getEnvBool("REDIS_EXPORTER_REDACT_CONFIG_METRICS", true), "Whether to redact config settings that include potentially sensitive information like passwords")
@@ -186,6 +188,8 @@ func main() {
 			IsTile38:                       *isTile38,
 			IsCluster:                      *isCluster,
 			InclModulesMetrics:             *inclModulesMetrics,
+			InclAofFileSize:                *inclAofFileSize,
+			OverrideAofFilePath:            *overrideAofFilePath,
 			ExportClientList:               *exportClientList,
 			ExportClientsInclPort:          *exportClientPort,
 			SkipTLSVerification:            *skipTLSVerification,



Test patch:
diff --git a/exporter/aof_test.go b/exporter/aof_test.go
new file mode 100644
index 0000000..6be6d3c
--- /dev/null
+++ b/exporter/aof_test.go
@@ -0,0 +1,61 @@
+package exporter
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestAof(t *testing.T) {
+	if os.Getenv("TEST_AOF_FILE_SIZE_URL") == "" {
+		t.Skipf("TEST_AOF_FILE_SIZE_URL not set - skipping")
+	}
+
+	tsts := []struct {
+		addr              string
+		enableAofFileSize bool
+		wantAofFileSize   bool
+	}{
+		{addr: os.Getenv("TEST_AOF_FILE_SIZE_URL"), enableAofFileSize: true, wantAofFileSize: true},
+		{addr: os.Getenv("TEST_AOF_FILE_SIZE_URL"), enableAofFileSize: false, wantAofFileSize: false},
+	}
+
+	for _, tst := range tsts {
+		e, _ := NewRedisExporter(tst.addr, Options{InclAofFileSize: tst.enableAofFileSize, OverrideAofFilePath: "../data/appendonlydir"})
+
+		chM := make(chan prometheus.Metric)
+
+		go func() {
+			e.Collect(chM)
+			close(chM)
+		}()
+
+		wantedMetrics := map[string]bool{
+			"aof_file_size_bytes": false,
+		}
+
+		for m := range chM {
+			for want := range wantedMetrics {
+				if strings.Contains(m.Desc().String(), want) {
+					wantedMetrics[want] = true
+				}
+			}
+		}
+
+		if tst.wantAofFileSize {
+			for want, found := range wantedMetrics {
+				if !found {
+					t.Errorf("%s was *not* found in AOF metrics but expected", want)
+				}
+			}
+		} else if !tst.wantAofFileSize {
+			for want, found := range wantedMetrics {
+				if found {
+					t.Errorf("%s was *found* in AOF metrics but *not* expected", want)
+				}
+			}
+		}
+	}
+}