Repository: minio/minio (language: go)

Issue:
Opening /proc/[pid]/io: permission denied
I'm trying to run minio in distributed mode - 8 nodes with Prometheus monitoring. Everything works fine, got statistics on dashboard etc. except that every node got repeating error in log file:

```bash
API: SYSTEM()
Time: 16:39:32 CET 02/02/2022
DeploymentID: 
Error: open /proc/1074/io: permission denied (*fs.PathError)
       8: internal/logger/logonce.go:52:logger.(*logOnceType).logOnceIf()
       7: internal/logger/logonce.go:92:logger.LogOnceIf()
       6: cmd/metrics-v2.go:885:cmd.getMinioProcMetrics.func1()
       5: cmd/metrics-v2.go:211:cmd.cachedRead.func1.1()
       4: cmd/utils.go:887:cmd.(*timedValue).Get()
       3: cmd/metrics-v2.go:218:cmd.cachedRead()
       2: cmd/metrics-v2.go:1728:cmd.populateAndPublish()
       1: cmd/metrics-v2.go:1694:cmd.ReportMetrics.func1()
```

When I check permissions on /proc for running service:

```bash
● minio.service - MinIO
   Loaded: loaded (/etc/systemd/system/minio.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2022-02-02 10:09:08 CET; 7h ago
     Docs: https://docs.min.io
  Process: 1068 ExecStartPre=/bin/bash -c if [ -z "${MINIO_VOLUMES}" ]; then echo "Variable MINIO_VOLUMES not set in /etc/default/minio"; exit 1; fi (code=exited, status=0/SUCCESS)
 Main PID: 1074 (minio)
    Tasks: 14
   Memory: 4.1G
   CGroup: /system.slice/minio.service
           └─1074 /usr/local/bin/minio server --address mds-1.local:9000 --console-address mds.local:443 --certs-dir /etc/ssl/minio https://mds-{1...8}.local/ds1/minio
```

Only few got owner as minio-user and **io** got permisison to be read only by root

```bash
dr-xr-xr-x.   9 minio-user minio-user 0 Feb  2 16:36 .
dr-xr-xr-x. 214 root       root       0 Feb  2 10:08 ..
-r--r--r--.   1 root       root       0 Feb  2 17:16 arch_status
dr-xr-xr-x.   2 minio-user minio-user 0 Feb  2 16:39 attr
-r--------.   1 root       root       0 Feb  2 17:16 auxv
-r--r--r--.   1 root       root       0 Feb  2 16:39 cgroup
--w-------.   1 root       root       0 Feb  2 17:16 clear_refs
-r--r--r--.   1 root       root       0 Feb  2 16:36 cmdline
-rw-r--r--.   1 root       root       0 Feb  2 16:39 comm
-rw-r--r--.   1 root       root       0 Feb  2 17:16 coredump_filter
-r--r--r--.   1 root       root       0 Feb  2 17:16 cpuset
lrwxrwxrwx.   1 root       root       0 Feb  2 17:16 cwd -> /usr/local
-r--------.   1 root       root       0 Feb  2 17:16 environ
lrwxrwxrwx.   1 root       root       0 Feb  2 16:39 exe -> /usr/local/bin/minio
dr-x------.   2 root       root       0 Feb  2 16:36 fd
dr-x------.   2 root       root       0 Feb  2 17:16 fdinfo
-rw-r--r--.   1 root       root       0 Feb  2 17:16 gid_map
-r--------.   1 root       root       0 Feb  2 16:36 io
-r--r--r--.   1 root       root       0 Feb  2 16:36 limits
-rw-r--r--.   1 root       root       0 Feb  2 16:39 loginuid
dr-x------.   2 root       root       0 Feb  2 17:16 map_files
-r--r--r--.   1 root       root       0 Feb  2 17:16 maps
-rw-------.   1 root       root       0 Feb  2 17:16 mem
-r--r--r--.   1 root       root       0 Feb  2 17:16 mountinfo
-r--r--r--.   1 root       root       0 Feb  2 17:16 mounts
-r--------.   1 root       root       0 Feb  2 17:16 mountstats
dr-xr-xr-x.  53 minio-user minio-user 0 Feb  2 17:16 net
dr-x--x--x.   2 root       root       0 Feb  2 17:16 ns
-r--r--r--.   1 root       root       0 Feb  2 17:16 numa_maps
-rw-r--r--.   1 root       root       0 Feb  2 17:16 oom_adj
-r--r--r--.   1 root       root       0 Feb  2 17:16 oom_score
-rw-r--r--.   1 root       root       0 Feb  2 17:16 oom_score_adj
-r--------.   1 root       root       0 Feb  2 17:16 pagemap
-r--------.   1 root       root       0 Feb  2 17:16 personality
-rw-r--r--.   1 root       root       0 Feb  2 17:16 projid_map
lrwxrwxrwx.   1 root       root       0 Feb  2 17:16 root -> /
-rw-r--r--.   1 root       root       0 Feb  2 17:16 sched
-r--r--r--.   1 root       root       0 Feb  2 17:16 schedstat
-r--r--r--.   1 root       root       0 Feb  2 16:39 sessionid
-rw-r--r--.   1 root       root       0 Feb  2 17:16 setgroups
-r--r--r--.   1 root       root       0 Feb  2 17:16 smaps
-r--r--r--.   1 root       root       0 Feb  2 17:16 smaps_rollup
-r--------.   1 root       root       0 Feb  2 17:16 stack
-r--r--r--.   1 root       root       0 Feb  2 16:58 stat
-r--r--r--.   1 root       root       0 Feb  2 17:16 statm
-r--r--r--.   1 root       root       0 Feb  2 16:36 status
-r--------.   1 root       root       0 Feb  2 17:16 syscall
dr-xr-xr-x.  16 minio-user minio-user 0 Feb  2 17:16 task
-r--r--r--.   1 root       root       0 Feb  2 17:16 timers
-rw-rw-rw-.   1 root       root       0 Feb  2 17:16 timerslack_ns
-rw-r--r--.   1 root       root       0 Feb  2 17:16 uid_map
-r--r--r--.   1 root       root       0 Feb  2 17:16 wchan

```

## Your Environment
Minio service file:

```
[Unit]
Description=MinIO
Documentation=https://docs.min.io
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
WorkingDirectory=/usr/local/

User=minio-user
Group=minio-user
ProtectProc=invisible

EnvironmentFile=/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES

# Let systemd restart this service always
Restart=always

# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=1048576

# Specifies the maximum number of threads this process can create
TasksMax=infinity

# Disable timeout logic and wait until process is stopped
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target

# Built for ${project.name}-${project.version} (${project.name})
```


* Version used: RELEASE.2021-11-24T23-19-33Z
* Operating System: Linux 5.4.17


Discussion / hints:
Please upgrade to latest release..
@harshavardhana sory my bad:

```bash
minio version RELEASE.2022-02-01T18-00-14Z
```

But still the same:

````bash
API: SYSTEM()
Time: 17:14:32 UTC 02/02/2022
DeploymentID:
Error: open /proc/13400/io: permission denied (*fs.PathError)
	7: internal/logger/logonce.go:52:logger.(*logOnceType).logOnceIf()
	6: internal/logger/logonce.go:92:logger.LogOnceIf()
	5: cmd/metrics-v2.go:958:cmd.getMinioProcMetrics.func1()
	4: cmd/metrics-v2.go:229:cmd.(*MetricsGroup).RegisterRead.func1.1()
	3: cmd/metrics-v2.go:262:cmd.(*MetricsGroup).Get()
	2: cmd/metrics-v2.go:1832:cmd.populateAndPublish()
	1: cmd/metrics-v2.go:1797:cmd.ReportMetrics.func1()
````

Permissions:

```bash
dr-xr-xr-x.   9 minio-user minio-user 0 Feb  2 18:14 .
dr-xr-xr-x. 215 root       root       0 Feb  2 10:09 ..
-r--r--r--.   1 root       root       0 Feb  2 18:29 arch_status
dr-xr-xr-x.   2 minio-user minio-user 0 Feb  2 18:14 attr
-r--------.   1 root       root       0 Feb  2 18:29 auxv
-r--r--r--.   1 root       root       0 Feb  2 18:14 cgroup
--w-------.   1 root       root       0 Feb  2 18:29 clear_refs
-r--r--r--.   1 root       root       0 Feb  2 18:14 cmdline
-rw-r--r--.   1 root       root       0 Feb  2 18:14 comm
-rw-r--r--.   1 root       root       0 Feb  2 18:29 coredump_filter
-r--r--r--.   1 root       root       0 Feb  2 18:14 cpuset
lrwxrwxrwx.   1 root       root       0 Feb  2 18:29 cwd -> /usr/local
-r--------.   1 root       root       0 Feb  2 18:29 environ
lrwxrwxrwx.   1 root       root       0 Feb  2 18:14 exe -> /usr/local/bin/minio
dr-x------.   2 root       root       0 Feb  2 18:14 fd
dr-x------.   2 root       root       0 Feb  2 18:29 fdinfo
-rw-r--r--.   1 root       root       0 Feb  2 18:29 gid_map
-r--------.   1 root       root       0 Feb  2 18:14 io
-r--r--r--.   1 root       root       0 Feb  2 18:14 limits
-rw-r--r--.   1 root       root       0 Feb  2 18:14 loginuid
dr-x------.   2 root       root       0 Feb  2 18:29 map_files
-r--r--r--.   1 root       root       0 Feb  2 18:29 maps
-rw-------.   1 root       root       0 Feb  2 18:29 mem
-r--r--r--.   1 root       root       0 Feb  2 18:29 mountinfo
-r--r--r--.   1 root       root       0 Feb  2 18:14 mounts
-r--------.   1 root       root       0 Feb  2 18:29 mountstats
dr-xr-xr-x.  53 minio-user minio-user 0 Feb  2 18:29 net
dr-x--x--x.   2 root       root       0 Feb  2 18:29 ns
-r--r--r--.   1 root       root       0 Feb  2 18:29 numa_maps
-rw-r--r--.   1 root       root       0 Feb  2 18:29 oom_adj
-r--r--r--.   1 root       root       0 Feb  2 18:29 oom_score
-rw-r--r--.   1 root       root       0 Feb  2 18:29 oom_score_adj
-r--------.   1 root       root       0 Feb  2 18:29 pagemap
-r--------.   1 root       root       0 Feb  2 18:29 personality
-rw-r--r--.   1 root       root       0 Feb  2 18:29 projid_map
lrwxrwxrwx.   1 root       root       0 Feb  2 18:29 root -> /
-rw-r--r--.   1 root       root       0 Feb  2 18:29 sched
-r--r--r--.   1 root       root       0 Feb  2 18:29 schedstat
-r--r--r--.   1 root       root       0 Feb  2 18:14 sessionid
-rw-r--r--.   1 root       root       0 Feb  2 18:29 setgroups
-r--r--r--.   1 root       root       0 Feb  2 18:29 smaps
-r--r--r--.   1 root       root       0 Feb  2 18:29 smaps_rollup
-r--------.   1 root       root       0 Feb  2 18:29 stack
-r--r--r--.   1 root       root       0 Feb  2 18:14 stat
-r--r--r--.   1 root       root       0 Feb  2 18:29 statm
-r--r--r--.   1 root       root       0 Feb  2 18:14 status
-r--------.   1 root       root       0 Feb  2 18:29 syscall
dr-xr-xr-x.  14 minio-user minio-user 0 Feb  2 18:29 task
-r--r--r--.   1 root       root       0 Feb  2 18:29 timers
-rw-rw-rw-.   1 root       root       0 Feb  2 18:29 timerslack_ns
-rw-r--r--.   1 root       root       0 Feb  2 18:29 uid_map
-r--r--r--.   1 root       root       0 Feb  2 18:29 wchan
```

Same service file. 

> 

Thanks you can remove `ProtectProc=invisible` this should allow them to be readable - we can also ignore the error on our end and never log. 

Fix patch:
diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go
index 47769461d39dc..1f4bc39cccc8d 100644
--- a/cmd/metrics-v2.go
+++ b/cmd/metrics-v2.go
@@ -942,94 +942,109 @@ func getMinioProcMetrics() *MetricsGroup {
 			logger.LogOnceIf(ctx, err, nodeMetricNamespace)
 			return
 		}
-		var openFDs int
-		openFDs, err = p.FileDescriptorsLen()
-		if err != nil {
-			logger.LogOnceIf(ctx, err, getMinioFDOpenMD())
-			return
+
+		openFDs, _ := p.FileDescriptorsLen()
+		l, _ := p.Limits()
+		io, _ := p.IO()
+		stat, _ := p.Stat()
+		startTime, _ := stat.StartTime()
+
+		if openFDs > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinioFDOpenMD(),
+					Value:       float64(openFDs),
+				},
+			)
 		}
-		l, err := p.Limits()
-		if err != nil {
-			logger.LogOnceIf(ctx, err, getMinioFDLimitMD())
-			return
+
+		if l.OpenFiles > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinioFDLimitMD(),
+					Value:       float64(l.OpenFiles),
+				})
 		}
-		io, err := p.IO()
-		if err != nil {
-			logger.LogOnceIf(ctx, err, ioSubsystem)
-			return
+
+		if io.SyscR > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessSysCallRMD(),
+					Value:       float64(io.SyscR),
+				})
 		}
-		stat, err := p.Stat()
-		if err != nil {
-			logger.LogOnceIf(ctx, err, processSubsystem)
-			return
+
+		if io.SyscW > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessSysCallWMD(),
+					Value:       float64(io.SyscW),
+				})
 		}
-		startTime, err := stat.StartTime()
-		if err != nil {
-			logger.LogOnceIf(ctx, err, startTime)
-			return
+
+		if io.ReadBytes > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinioProcessIOReadBytesMD(),
+					Value:       float64(io.ReadBytes),
+				})
 		}
 
-		metrics = append(metrics,
-			Metric{
-				Description: getMinioFDOpenMD(),
-				Value:       float64(openFDs),
-			},
-		)
-		metrics = append(metrics,
-			Metric{
-				Description: getMinioFDLimitMD(),
-				Value:       float64(l.OpenFiles),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinIOProcessSysCallRMD(),
-				Value:       float64(io.SyscR),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinIOProcessSysCallWMD(),
-				Value:       float64(io.SyscW),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinioProcessIOReadBytesMD(),
-				Value:       float64(io.ReadBytes),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinioProcessIOWriteBytesMD(),
-				Value:       float64(io.WriteBytes),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinioProcessIOReadCachedBytesMD(),
-				Value:       float64(io.RChar),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinioProcessIOWriteCachedBytesMD(),
-				Value:       float64(io.WChar),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinIOProcessStartTimeMD(),
-				Value:       startTime,
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinIOProcessUptimeMD(),
-				Value:       time.Since(globalBootTime).Seconds(),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinIOProcessResidentMemory(),
-				Value:       float64(stat.ResidentMemory()),
-			})
-		metrics = append(metrics,
-			Metric{
-				Description: getMinIOProcessCPUTime(),
-				Value:       stat.CPUTime(),
-			})
+		if io.WriteBytes > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinioProcessIOWriteBytesMD(),
+					Value:       float64(io.WriteBytes),
+				})
+		}
+
+		if io.RChar > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinioProcessIOReadCachedBytesMD(),
+					Value:       float64(io.RChar),
+				})
+		}
+
+		if io.WChar > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinioProcessIOWriteCachedBytesMD(),
+					Value:       float64(io.WChar),
+				})
+		}
+
+		if startTime > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessStartTimeMD(),
+					Value:       startTime,
+				})
+		}
+
+		if !globalBootTime.IsZero() {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessUptimeMD(),
+					Value:       time.Since(globalBootTime).Seconds(),
+				})
+		}
+
+		if stat.ResidentMemory() > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessResidentMemory(),
+					Value:       float64(stat.ResidentMemory()),
+				})
+		}
+
+		if stat.CPUTime() > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessCPUTime(),
+					Value:       stat.CPUTime(),
+				})
+		}
 		return
 	})
 	return mg


Test patch: