Repository: nats-io/nats-server (go)

Issue:
Stream delete can fail for non empty directory
During concurrent consumer create/delete and stream create/delete it can happen that a stream directory during delete is not empty 

```
stream delete failed: unlinkat /var/folders/2p/lsd279293x7618xcjfl2bc380000gn/T/jstest632941891/jetstream/$G/streams/ELECTION: directory not empty (10050)
```

Pull request description:
[FIXED] #2417
Signed-off-by: Derek Collison <derek@nats.io>

Resolves #2417 

/cc @nats-io/core


Patch:
diff --git a/server/filestore.go b/server/filestore.go
index 30f04d87..4d4d3331 100644
--- a/server/filestore.go
+++ b/server/filestore.go
@@ -4145,7 +4145,19 @@ func (fs *fileStore) Delete() error {
 	if err := fs.Stop(); err != nil {
 		return err
 	}
-	return os.RemoveAll(fs.fcfg.StoreDir)
+
+	err := os.RemoveAll(fs.fcfg.StoreDir)
+	if err == nil {
+		return nil
+	}
+	ttl := time.Now().Add(time.Second)
+	for time.Now().Before(ttl) {
+		time.Sleep(10 * time.Millisecond)
+		if err = os.RemoveAll(fs.fcfg.StoreDir); err == nil {
+			return nil
+		}
+	}
+	return err
 }
 
 // Lock should be held.


Test patch:
diff --git a/server/filestore_test.go b/server/filestore_test.go
index 5a59506f..79d2760a 100644
--- a/server/filestore_test.go
+++ b/server/filestore_test.go
@@ -2675,6 +2675,41 @@ func TestFileStoreStreamStateDeleted(t *testing.T) {
 	}
 }
 
+// We have reports that sometimes under load a stream could complain about a storage directory
+// not being empty.
+func TestFileStoreStreamDeleteDirNotEmpty(t *testing.T) {
+	storeDir := createDir(t, JetStreamStoreDir)
+	defer removeDir(t, storeDir)
+
+	fs, err := newFileStore(FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "zzz", Storage: FileStorage})
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer fs.Stop()
+
+	subj, toStore := "foo", uint64(10)
+	for i := uint64(1); i <= toStore; i++ {
+		msg := []byte(fmt.Sprintf("[%08d] Hello World!", i))
+		if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil {
+			t.Fatalf("Error storing msg: %v", err)
+		}
+	}
+
+	ready := make(chan bool)
+	go func() {
+		g := path.Join(storeDir, "g")
+		ready <- true
+		for i := 0; i < 100; i++ {
+			ioutil.WriteFile(g, []byte("OK"), defaultFilePerms)
+		}
+	}()
+
+	<-ready
+	if err := fs.Delete(); err != nil {
+		t.Fatalf("Delete returned an error: %v", err)
+	}
+}
+
 func TestFileStoreConsumerPerf(t *testing.T) {
 	// Comment out to run, holding place for now.
 	t.SkipNow()