Repository: minio/minio (language: go)

Issue:
Minio on macos + docker no longer starts with RELEASE.2022-06-02T02-11-04Z
## Expected Behavior
Minio should start with `minio:latest` / `minio:RELEASE.2022-06-02T02-11-04Z`

## Current Behavior

As per older issues (https://github.com/minio/minio/issues/9554, https://github.com/minio/minio/issues/10277), with `RELEASE.2022-06-02T02-11-04Z` minio fails to start and complains about requiring `O_DIRECT` support.


As stated in https://docs.min.io/minio/baremetal/installation/deploy-minio-single-node-single-drive.html#pre-existing-data, it seems single drive mode now uses erasure coding (which requires `O_DIRECT`?). Where before the recommendation was to ensure you run in filesystem / single drive mode for macos + docker, it seems this may no longer be possible? As stated in that link it suggests running an older version of minio.

## Possible Solution
Run older version (e.g. `RELEASE.2022-05-26T05-48-41Z`) of minio (precluding future updates)

## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
<!--- and make sure you have followed https://github.com/minio/minio/tree/release/docs/debugging to capture relevant logs -->

1. Start `minio` on macos + docker with version `RELEASE.2022-06-02T02-11-04Z`
2. Minio will error on startup complaining about requiring `O_DIRECT` support.

## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->

## Regression

See versions above.

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used (`minio --version`): `RELEASE.2022-06-02T02-11-04Z`
* Server setup and configuration: minio invoked with `minio server --console-address :9001 /data`
* Operating System and version (`uname -a`): `Darwin Kernel Version 21.5.0: Tue Apr 26 21:08:29 PDT 2022; root:xnu-8020.121.3~4/RELEASE_ARM64_T8101 arm64`



Discussion / hints:
@malcolmstill we need a backend that supports O_DIRECT we cannot do without that.

We suggest that you use binary directly on macOS host as workaround for now.

Will see if this can be disabled on macOS docker instances.

Fix patch:
diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go
index 5851dc861f8fd..ea375528f0b1e 100644
--- a/cmd/xl-storage.go
+++ b/cmd/xl-storage.go
@@ -94,8 +94,8 @@ type xlStorage struct {
 	endpoint Endpoint
 
 	globalSync bool
-
-	rootDisk bool
+	oDirect    bool // indicates if this disk supports ODirect
+	rootDisk   bool
 
 	diskID string
 
@@ -264,15 +264,27 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		filePath := pathJoin(s.diskPath, ".writable-check-"+uuid+".tmp")
 		w, err := s.openFileDirect(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL)
 		if err != nil {
-			return s, err
+			// relax single drive mode without O_DIRECT support
+			if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+				s.oDirect = false
+			} else {
+				return s, err
+			}
 		}
 		_, err = w.Write(alignedBuf)
 		w.Close()
 		if err != nil {
 			if isSysErrInvalidArg(err) {
-				return s, errUnsupportedDisk
+				err = errUnsupportedDisk
+			}
+			// relax single drive mode without O_DIRECT support
+			if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+				s.oDirect = false
+			} else {
+				return s, err
 			}
-			return s, err
+		} else {
+			s.oDirect = true
 		}
 		renameAll(filePath, pathJoin(s.diskPath, minioMetaTmpDeletedBucket, uuid))
 
@@ -289,6 +301,7 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		s.diskID = format.Erasure.This
 		s.formatLastCheck = time.Now()
 		s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
+		s.oDirect = true
 	}
 
 	// Success.
@@ -1372,7 +1385,13 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		return nil, time.Time{}, ctx.Err()
 	}
 
-	f, err := OpenFileDirectIO(filePath, readMode, 0o666)
+	odirectEnabled := !globalAPIConfig.isDisableODirect() && s.oDirect
+	var f *os.File
+	if odirectEnabled {
+		f, err = OpenFileDirectIO(filePath, readMode, 0o666)
+	} else {
+		f, err = OpenFile(filePath, readMode, 0o666)
+	}
 	if err != nil {
 		if osIsNotExist(err) {
 			// Check if the object doesn't exist because its bucket
@@ -1662,7 +1681,14 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, err
 	}
 
-	file, err := OpenFileDirectIO(filePath, readMode, 0o666)
+	odirectEnabled := !globalAPIConfig.isDisableODirect() && s.oDirect
+
+	var file *os.File
+	if odirectEnabled {
+		file, err = OpenFileDirectIO(filePath, readMode, 0o666)
+	} else {
+		file, err = OpenFile(filePath, readMode, 0o666)
+	}
 	if err != nil {
 		switch {
 		case osIsNotExist(err):
@@ -1706,7 +1732,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	}
 
 	alignment := offset%xioutil.DirectioAlignSize == 0
-	if !alignment || globalAPIConfig.isDisableODirect() {
+	if !alignment && odirectEnabled {
 		if err = disk.DisableDirectIO(file); err != nil {
 			file.Close()
 			return nil, err
@@ -1736,13 +1762,11 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		io.Reader
 		io.Closer
 	}{Reader: io.LimitReader(diskHealthReader(ctx, or), length), Closer: closeWrapper(func() error {
-		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
+		if (!alignment || offset+length%xioutil.DirectioAlignSize != 0) && odirectEnabled {
 			// invalidate page-cache for unaligned reads.
-			if !globalAPIConfig.isDisableODirect() {
-				// skip removing from page-cache only
-				// if O_DIRECT was disabled.
-				disk.FadviseDontNeed(file)
-			}
+			// skip removing from page-cache only
+			// if O_DIRECT was disabled.
+			disk.FadviseDontNeed(file)
 		}
 		return or.Close()
 	})}
@@ -1814,7 +1838,13 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		return osErrToFileErr(err)
 	}
 
-	w, err := OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	odirectEnabled := s.oDirect
+	var w *os.File
+	if odirectEnabled {
+		w, err = OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	} else {
+		w, err = OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	}
 	if err != nil {
 		return osErrToFileErr(err)
 	}
@@ -1834,7 +1864,12 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
-	written, err := xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
+	var written int64
+	if odirectEnabled {
+		written, err = xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
+	} else {
+		written, err = io.CopyBuffer(diskHealthWriter(ctx, w), r, *bufp)
+	}
 	if err != nil {
 		return err
 	}


Test patch: