Repository: apache/nuttx
Pull request #16431: Resolving the issue of fdcheck reporting errors when an epoll file descriptor (fd) is closed
Author: login: Donny9
Labels: name: Area: File System
description: File System issues
name: Size: M
description: The size of the change in this PR is medium
Created: 2025-05-23T08:38:47Z | Merged: 2025-05-26T10:49:45Z


## Summary

Resolving the issue of fdcheck reporting errors when an epoll file descriptor (fd) is closed.

The primary cause of the error is as follows: 
When a task exits by calling exit, it closes all file descriptors (fds) associated with the current task. 
When the epoll fd is closed, if there are still fds in the epoll queue that have values greater than the epoll fd, these fds may be closed first. This can lead to fdcheck failures when call epoll_close
To resolve this, we should use a file-based approach instead of directly managing fds in epoll.

## Impact

fix epoll fdcheck issue

## Testing

open vela monkey





Diff:
diff --git a/fs/vfs/fs_epoll.c b/fs/vfs/fs_epoll.c
index dc78f935d28c0..f61cf9f631e61 100644
--- a/fs/vfs/fs_epoll.c
+++ b/fs/vfs/fs_epoll.c
@@ -56,6 +56,7 @@ struct epoll_node_s
   epoll_data_t             data;
   bool                     notified;
   struct pollfd            pfd;
+  FAR struct file         *filep;
   FAR struct epoll_head_s *eph;
 };
 
@@ -198,7 +199,8 @@ static int epoll_do_close(FAR struct file *filep)
       nxmutex_destroy(&eph->lock);
       list_for_every_entry(&eph->setup, epn, epoll_node_t, node)
         {
-          poll_fdsetup(epn->pfd.fd, &epn->pfd, false);
+          file_poll(epn->filep, &epn->pfd, false);
+          fs_putfilep(epn->filep);
         }
 
       list_for_every_entry_safe(&eph->extend, epn, tmp, epoll_node_t, node)
@@ -302,11 +304,11 @@ static int epoll_setup(FAR epoll_head_t *eph)
 
       epn->notified    = false;
       epn->pfd.revents = 0;
-      ret = poll_fdsetup(epn->pfd.fd, &epn->pfd, true);
+      ret = file_poll(epn->filep, &epn->pfd, true);
       if (ret < 0)
         {
-          ferr("epoll setup failed, fd=%d, events=%08" PRIx32 ", ret=%d\n",
-               epn->pfd.fd, epn->pfd.events, ret);
+          ferr("epoll setup failed, filep=%p, events=%08" PRIx32 ", "
+               "ret=%d\n", epn->filep, epn->pfd.events, ret);
           break;
         }
 
@@ -356,7 +358,7 @@ static int epoll_teardown(FAR epoll_head_t *eph, FAR struct epoll_event *evs,
 
       /* Teradown all the notified fd */
 
-      poll_fdsetup(epn->pfd.fd, &epn->pfd, false);
+      file_poll(epn->filep, &epn->pfd, false);
       list_delete(&epn->node);
 
       if (epn->pfd.revents != 0 && i < maxevents)
@@ -488,6 +490,7 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
   eph = epoll_head_from_fd(epfd, &filep);
   if (eph == NULL)
     {
+      set_errno(EBADF);
       return ERROR;
     }
 
@@ -565,9 +568,17 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
         epn->pfd.cb      = epoll_default_cb;
         epn->pfd.revents = 0;
 
-        ret = poll_fdsetup(fd, &epn->pfd, true);
+        ret = fs_getfilep(fd, &epn->filep);
+        if (ret < 0)
+          {
+            list_add_tail(&eph->free, &epn->node);
+            goto err;
+          }
+
+        ret = file_poll(epn->filep, &epn->pfd, true);
         if (ret < 0)
           {
+            fs_putfilep(epn->filep);
             list_add_tail(&eph->free, &epn->node);
             goto err;
           }
@@ -581,7 +592,8 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
           {
             if (epn->pfd.fd == fd)
               {
-                poll_fdsetup(fd, &epn->pfd, false);
+                file_poll(epn->filep, &epn->pfd, false);
+                fs_putfilep(epn->filep);
                 list_delete(&epn->node);
                 list_add_tail(&eph->free, &epn->node);
                 goto out;
@@ -592,6 +604,7 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
           {
             if (epn->pfd.fd == fd)
               {
+                fs_putfilep(epn->filep);
                 list_delete(&epn->node);
                 list_add_tail(&eph->free, &epn->node);
                 goto out;
@@ -602,6 +615,7 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
           {
             if (epn->pfd.fd == fd)
               {
+                fs_putfilep(epn->filep);
                 list_delete(&epn->node);
                 list_add_tail(&eph->free, &epn->node);
                 goto out;
@@ -618,15 +632,14 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
               {
                 if (epn->pfd.events != (ev->events | POLLALWAYS))
                   {
-                    poll_fdsetup(fd, &epn->pfd, false);
+                    file_poll(epn->filep, &epn->pfd, false);
 
                     epn->notified    = false;
                     epn->data        = ev->data;
                     epn->pfd.events  = ev->events | POLLALWAYS;
-                    epn->pfd.fd      = fd;
                     epn->pfd.revents = 0;
 
-                    ret = poll_fdsetup(fd, &epn->pfd, true);
+                    ret = file_poll(epn->filep, &epn->pfd, true);
                     if (ret < 0)
                       {
                         goto err;
@@ -646,10 +659,9 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
                     epn->notified    = false;
                     epn->data        = ev->data;
                     epn->pfd.events  = ev->events | POLLALWAYS;
-                    epn->pfd.fd      = fd;
                     epn->pfd.revents = 0;
 
-                    ret = poll_fdsetup(fd, &epn->pfd, true);
+                    ret = file_poll(epn->filep, &epn->pfd, true);
                     if (ret < 0)
                       {
                         goto err;
@@ -670,10 +682,9 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
                 epn->notified    = false;
                 epn->data        = ev->data;
                 epn->pfd.events  = ev->events | POLLALWAYS;
-                epn->pfd.fd      = fd;
                 epn->pfd.revents = 0;
 
-                ret = poll_fdsetup(fd, &epn->pfd, true);
+                ret = file_poll(epn->filep, &epn->pfd, true);
                 if (ret < 0)
                   {
                     goto err;
diff --git a/fs/vfs/fs_poll.c b/fs/vfs/fs_poll.c
index f309c655e9a75..f53f5609815b9 100644
--- a/fs/vfs/fs_poll.c
+++ b/fs/vfs/fs_poll.c
@@ -79,8 +79,24 @@ static inline void poll_teardown(FAR struct pollfd *fds, nfds_t nfds,
     {
       if (fds[i].fd >= 0)
         {
-          int status = poll_fdsetup(fds[i].fd, &fds[i], false);
-          if (status < 0)
+          FAR struct file *filep;
+          int ret;
+
+          ret = fs_getfilep(fds[i].fd, &filep);
+          if (ret >= 0)
+            {
+              ret = file_poll(filep, &fds[i], false);
+
+              /* Calling putfilep twice to ensure reference counting
+               * for filep remains consistent with its state for
+               * before the poll.
+               */
+
+              fs_putfilep(filep);
+              fs_putfilep(filep);
+            }
+
+          if (ret < 0)
             {
               fds[i].revents |= POLLERR;
             }
@@ -144,10 +160,26 @@ static inline int poll_setup(FAR struct pollfd *fds, nfds_t nfds,
 
       if (fds[i].fd >= 0)
         {
-          ret = poll_fdsetup(fds[i].fd, &fds[i], true);
+          FAR struct file *filep;
+          int num = i;
+
+          ret = fs_getfilep(fds[i].fd, &filep);
+          if (ret < 0)
+            {
+              num -= 1;
+            }
+          else
+            {
+              ret = file_poll(filep, &fds[i], true);
+            }
+
           if (ret < 0)
             {
-              poll_teardown(fds, i, &count);
+              if (num >= 0)
+                {
+                  poll_teardown(fds, num, &count);
+                }
+
               fds[i].revents |= POLLERR;
               fds[i].arg = NULL;
               fds[i].cb = NULL;
@@ -192,38 +224,6 @@ static void poll_cleanup(FAR void *arg)
  * Public Functions
  ****************************************************************************/
 
-/****************************************************************************
- * Name: poll_fdsetup
- *
- * Description:
- *   Configure (or unconfigure) one file/socket descriptor for the poll
- *   operation.  If fds and sem are non-null, then the poll is being setup.
- *   if fds and sem are NULL, then the poll is being torn down.
- *
- ****************************************************************************/
-
-int poll_fdsetup(int fd, FAR struct pollfd *fds, bool setup)
-{
-  FAR struct file *filep;
-  int ret;
-
-  /* Get the file pointer corresponding to this file descriptor */
-
-  ret = fs_getfilep(fd, &filep);
-  if (ret < 0)
-    {
-      return ret;
-    }
-
-  DEBUGASSERT(filep != NULL);
-
-  /* Let file_poll() do the rest */
-
-  ret = file_poll(filep, fds, setup);
-  fs_putfilep(filep);
-  return ret;
-}
-
 /****************************************************************************
  * Name: poll_default_cb
  *
@@ -309,9 +309,8 @@ void poll_notify(FAR struct pollfd **afds, int nfds, pollevent_t eventset)
  * Name: file_poll
  *
  * Description:
- *   Low-level poll operation based on struct file.  This is used both to (1)
- *   support detached file, and also (2) by poll_fdsetup() to perform all
- *   normal operations on file descriptors.
+ *   Low-level poll operation based on struct file.  This is used to
+ *   support detached file.
  *
  * Input Parameters:
  *   file     File structure instance
diff --git a/include/nuttx/fs/fs.h b/include/nuttx/fs/fs.h
index 156d67279a146..4e73f8b2c9741 100644
--- a/include/nuttx/fs/fs.h
+++ b/include/nuttx/fs/fs.h
@@ -1676,9 +1676,8 @@ int file_fcntl(FAR struct file *filep, int cmd, ...);
  * Name: file_poll
  *
  * Description:
- *   Low-level poll operation based on struct file.  This is used both to (1)
- *   support detached file, and also (2) by poll_fdsetup() to perform all
- *   normal operations on file descriptors.
+ *   Low-level poll operation based on struct file.  This is used to
+ *   support detached file.
  *
  * Input Parameters:
  *   file     File structure instance
diff --git a/include/sys/poll.h b/include/sys/poll.h
index 23690e8c13b62..4188ead31a900 100644
--- a/include/sys/poll.h
+++ b/include/sys/poll.h
@@ -154,7 +154,6 @@ int ppoll(FAR struct pollfd *fds, nfds_t nfds,
           FAR const struct timespec *timeout_ts,
           FAR const sigset_t *sigmask);
 
-int poll_fdsetup(int fd, FAR struct pollfd *fds, bool setup);
 void poll_default_cb(FAR struct pollfd *fds);
 void poll_notify(FAR struct pollfd **afds, int nfds, pollevent_t eventset);
 


Reviews:
user: login: xiaoxiang781216
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T08:51:53Z
user: login: pussuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T09:02:48Z
user: login: pussuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T09:03:20Z
user: login: Donny9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T09:37:04Z
user: login: Donny9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T09:41:38Z
user: login: pussuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T10:54:21Z
user: login: pussuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T10:56:21Z
user: login: GUIDINGLI
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T11:41:16Z
user: login: Donny9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T12:13:19Z
user: login: pussuw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T12:17:56Z
user: login: Donny9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T12:24:05Z
user: login: xiaoxiang781216
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T04:23:06Z
user: login: jerpelea
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T08:17:31Z
user: login: pussuw
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-26T09:50:02Z

Review comments:
diff_hunk: @@ -476,14 +479,32 @@ void files_putlist(FAR struct filelist *list)
    * because there should not be any references in this context.
    */
 
+#ifdef CONFIG_FS_REFCOUNT
+again:
+  loop = false;
+#endif
   for (i = list->fl_rows - 1; i >= 0; i--)
     {
       for (j = CONFIG_NFILE_DESCRIPTORS_PER_BLOCK - 1; j >= 0; j--)
         {
+#ifdef CONFIG_FS_REFCOUNT
+          if (fs_putfilep(&list->fl_files[i][j]) > 0)
path: fs/inode/fs_files.c
user: login: pussuw
body: I don't agree with this at all. Whomever holds the reference should release it. Now if the kernel needs the file still because it's doing something with it on another CPU it will crash.
created_at: 2025-05-23T09:02:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -482,11 +482,11 @@ void files_putlist(FAR struct filelist *list)
         {
           file_close(&list->fl_files[i][j]);
         }
+    }
 
-      if (i != 0)
-        {
-          fs_heap_free(list->fl_files[i]);
-        }
+  for (i = list->fl_rows - 1; i > 0; i--)
path: fs/inode/fs_files.c
user: login: pussuw
body: Looks like the change is redundant, the old code works as far as I can see ?
created_at: 2025-05-23T09:03:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -476,14 +479,32 @@ void files_putlist(FAR struct filelist *list)
    * because there should not be any references in this context.
    */
 
+#ifdef CONFIG_FS_REFCOUNT
+again:
+  loop = false;
+#endif
   for (i = list->fl_rows - 1; i >= 0; i--)
     {
       for (j = CONFIG_NFILE_DESCRIPTORS_PER_BLOCK - 1; j >= 0; j--)
         {
+#ifdef CONFIG_FS_REFCOUNT
+          if (fs_putfilep(&list->fl_files[i][j]) > 0)
path: fs/inode/fs_files.c
user: login: Donny9
body: @pussuw Hello, the call of this function occurs when a task exits. There are two key points that need to be clarified first:
1. All resources must be released when a task exits, including any opened file descriptors (fd).
2. If the reference count (refs) of the filep corresponding to an fd is greater than 1, it means that the filep is still being used somewhere, possibly by another duplicated fd.

Currently, we close all fds in the file_list in reverse order. 
For example:
Suppose the epoll fd is 5, and we add fd1=10, fd2=6, and fd3=3 to the epoll queue.
So the closing order would be: fd1 -> fd2 -> epoll fd -> fd3.

When we close the epoll fd, since it's part of the task exit process, there's no opportunity to execute epoll_ctl delete to remove fd1 and fd2. Therefore, when we call close(epoll_fd)->epoll_close, trying to find the filep through fd1 and fd2 will result in an error from fdcheck because they have already been closed.

This is why we're considering replacing the fds in epoll with file references. By doing so, when we close fd1 and fd2 for the first time, we only decrement their reference counts by 1 instead of actually closing them. 

However, we need to iterate through the for loop again to close them properly and release the resources.

"What do you mean by 'other CPUs'? Are you referring to scenarios where other CPUs access across cores through rpmsg services? In such cases, other CPUs would receive a POLLHUP event from this fd, and this event needs to be handled properly."
created_at: 2025-05-23T09:37:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -482,11 +482,11 @@ void files_putlist(FAR struct filelist *list)
         {
           file_close(&list->fl_files[i][j]);
         }
+    }
 
-      if (i != 0)
-        {
-          fs_heap_free(list->fl_files[i]);
-        }
+  for (i = list->fl_rows - 1; i > 0; i--)
path: fs/inode/fs_files.c
user: login: Donny9
body: @pussuw  No, you definitely haven't read my commit message carefully. 
The reason is that there are **many file descriptors (fds) that are interdependent,** and they may not all be on the same line in the list; they can span multiple lines. 
If, when closing one fd, you try to access another fd that has already been closed, but the line it was on may have already been freed, it could lead to a Use-After-Free (UAF) issue. 
Therefore, we need to release all the memory uniformly at the end.
created_at: 2025-05-23T09:41:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -482,11 +482,11 @@ void files_putlist(FAR struct filelist *list)
         {
           file_close(&list->fl_files[i][j]);
         }
+    }
 
-      if (i != 0)
-        {
-          fs_heap_free(list->fl_files[i]);
-        }
+  for (i = list->fl_rows - 1; i > 0; i--)
path: fs/inode/fs_files.c
user: login: pussuw
body: This problem will not exist after #16361 I think ?
created_at: 2025-05-23T10:54:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -476,14 +479,32 @@ void files_putlist(FAR struct filelist *list)
    * because there should not be any references in this context.
    */
 
+#ifdef CONFIG_FS_REFCOUNT
+again:
+  loop = false;
+#endif
   for (i = list->fl_rows - 1; i >= 0; i--)
     {
       for (j = CONFIG_NFILE_DESCRIPTORS_PER_BLOCK - 1; j >= 0; j--)
         {
+#ifdef CONFIG_FS_REFCOUNT
+          if (fs_putfilep(&list->fl_files[i][j]) > 0)
path: fs/inode/fs_files.c
user: login: pussuw
body: I mean another CPU can be performing a write/read/whatever operation while another CPU exit()'s the process. This is userspace racing against itself. The kernel must be able to endure this.
created_at: 2025-05-23T10:56:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -482,11 +482,11 @@ void files_putlist(FAR struct filelist *list)
         {
           file_close(&list->fl_files[i][j]);
         }
+    }
 
-      if (i != 0)
-        {
-          fs_heap_free(list->fl_files[i]);
-        }
+  for (i = list->fl_rows - 1; i > 0; i--)
path: fs/inode/fs_files.c
user: login: Donny9
body: > This problem will not exist after #16361 I think ?

@pussuw  You can construct a test case and try it out using the example I provided above, or I can test your patch over the weekend to see if it resolves this issue.
created_at: 2025-05-23T12:13:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -482,11 +482,11 @@ void files_putlist(FAR struct filelist *list)
         {
           file_close(&list->fl_files[i][j]);
         }
+    }
 
-      if (i != 0)
-        {
-          fs_heap_free(list->fl_files[i]);
-        }
+  for (i = list->fl_rows - 1; i > 0; i--)
path: fs/inode/fs_files.c
user: login: pussuw
body: Goes to next week for me
created_at: 2025-05-23T12:17:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -476,14 +479,32 @@ void files_putlist(FAR struct filelist *list)
    * because there should not be any references in this context.
    */
 
+#ifdef CONFIG_FS_REFCOUNT
+again:
+  loop = false;
+#endif
   for (i = list->fl_rows - 1; i >= 0; i--)
     {
       for (j = CONFIG_NFILE_DESCRIPTORS_PER_BLOCK - 1; j >= 0; j--)
         {
+#ifdef CONFIG_FS_REFCOUNT
+          if (fs_putfilep(&list->fl_files[i][j]) > 0)
path: fs/inode/fs_files.c
user: login: Donny9
body: > I mean another CPU can be performing a write/read/whatever operation while another CPU exit()'s the process. This is userspace racing against itself. The kernel must be able to endure this.

@pussuw 
The scenario you're describing should be: 
**A task contains multiple pthreads, where CPU A is scheduling one of the pthreads and is currently performing read/write operations, while another CPU B is executing the exit process for that task.** 

In this situation, we ensure in group_release that all child pthreads exit first via group_remove_children, and then the main task's exit is completed, releasing all file descriptors (fds). There are no issues with this; it's been done this way before, and it has no relation to the modifications in the current patch.
created_at: 2025-05-23T12:24:05Z
author_association: CONTRIBUTOR

PR comments:
user: login: pussuw
created_at: 2025-05-23T09:11:09Z
body: Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?
author_association: CONTRIBUTOR
user: login: Donny9
created_at: 2025-05-23T09:45:14Z
body: > Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?

@pussuw The implementation of fdcheck originally resides in libc https://github.com/apache/nuttx/blob/889a26db1d229ee462bc8ea075c2330dc5f79b94/libs/libc/misc/lib_fdcheck.c

But we need to perform the conversion from file descriptor (fd) to file structure pointer (filep) within the kernel, as this conversion cannot be done in user space. And, the parameter for system calls is an fd, nothing else, and the filep is also a kernel data structure.


author_association: CONTRIBUTOR
user: login: pussuw
created_at: 2025-05-23T10:55:11Z
body: > > Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?
> 
> @pussuw The implementation of fdcheck originally resides in libc https://github.com/apache/nuttx/blob/889a26db1d229ee462bc8ea075c2330dc5f79b94/libs/libc/misc/lib_fdcheck.c
> 
> But we need to perform the conversion from file descriptor (fd) to file structure pointer (filep) within the kernel, as this conversion cannot be done in user space. And, the parameter for system calls is an fd, nothing else, and the filep is also a kernel data structure.

Yes the problem here is that the data is in filep. After #16361 you could do this magic in userspace.
author_association: CONTRIBUTOR
user: login: jlaitine
created_at: 2025-05-23T11:03:13Z
body: I completely agree with @pussuw here! There is nice cleanup available for the FDs in PR #16361, which fixes the operation to be acc. to the posix.

Please don't inject debug functionality which should NOT be inside kernel in there. Integrate first the #16361, and then implement the debug functionality in where it belongs. That is, in the kernel-userspace interface, not directly in the kernel.

author_association: CONTRIBUTOR
user: login: Donny9
created_at: 2025-05-23T11:36:44Z
body: > > > Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?
> > 
> > 
> > @pussuw The implementation of fdcheck originally resides in libc https://github.com/apache/nuttx/blob/889a26db1d229ee462bc8ea075c2330dc5f79b94/libs/libc/misc/lib_fdcheck.c
> > But we need to perform the conversion from file descriptor (fd) to file structure pointer (filep) within the kernel, as this conversion cannot be done in user space. And, the parameter for system calls is an fd, nothing else, and the filep is also a kernel data structure.
> 
> Yes the problem here is that the data is in filep. After #16361 you could do this magic in userspace.

@pussuw  #16361  would definitely be the best if we could cover this type of scenario. 
However, the changes in this current Pull Request (PR) are quite extensive, so it's unlikely to be merged into the mainline quickly. There's a possibility of stability issues, so we should ensure its reliability through some testing. 
Therefore, I will pick it up internally and conduct LTP (Linux Test Project) functional testing, 24-hour Monkey stress testing, as well as functional testing on multiple platforms, before approving it.  

Therefore, our priority is to ensure that the current design is sound. You can continue to rework based on #16361 .
author_association: CONTRIBUTOR
user: login: pussuw
created_at: 2025-05-23T11:39:57Z
body: > > > > Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?
> > > 
> > > 
> > > @pussuw The implementation of fdcheck originally resides in libc https://github.com/apache/nuttx/blob/889a26db1d229ee462bc8ea075c2330dc5f79b94/libs/libc/misc/lib_fdcheck.c
> > > But we need to perform the conversion from file descriptor (fd) to file structure pointer (filep) within the kernel, as this conversion cannot be done in user space. And, the parameter for system calls is an fd, nothing else, and the filep is also a kernel data structure.
> > 
> > 
> > Yes the problem here is that the data is in filep. After #16361 you could do this magic in userspace.
> 
> @pussuw #16361 would definitely be the best if we could cover this type of scenario. However, the changes in this current Pull Request (PR) are quite extensive, so it's unlikely to be merged into the mainline quickly. There's a possibility of stability issues, so we should ensure its reliability through some testing. Therefore, I will pick it up internally and conduct LTP (Linux Test Project) functional testing, 24-hour Monkey stress testing, as well as functional testing on multiple platforms, before approving it.
> 
> Therefore, our priority is to ensure that the current design is sound. You can continue to rework based on #16361 .

Thanks, I do really prefer fixing the file descriptor subsystem like @xiaoxiang781216 suggested in #16326 . It is better to do a comprehensive fix that as far as I can see, will solve most of your problems as well.
author_association: CONTRIBUTOR
user: login: Donny9
created_at: 2025-05-23T12:09:27Z
body: > > > > > Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?
> > > > 
> > > > 
> > > > @pussuw The implementation of fdcheck originally resides in libc https://github.com/apache/nuttx/blob/889a26db1d229ee462bc8ea075c2330dc5f79b94/libs/libc/misc/lib_fdcheck.c
> > > > But we need to perform the conversion from file descriptor (fd) to file structure pointer (filep) within the kernel, as this conversion cannot be done in user space. And, the parameter for system calls is an fd, nothing else, and the filep is also a kernel data structure.
> > > 
> > > 
> > > Yes the problem here is that the data is in filep. After #16361 you could do this magic in userspace.
> > 
> > 
> > @pussuw #16361 would definitely be the best if we could cover this type of scenario. However, the changes in this current Pull Request (PR) are quite extensive, so it's unlikely to be merged into the mainline quickly. There's a possibility of stability issues, so we should ensure its reliability through some testing. Therefore, I will pick it up internally and conduct LTP (Linux Test Project) functional testing, 24-hour Monkey stress testing, as well as functional testing on multiple platforms, before approving it.
> > Therefore, our priority is to ensure that the current design is sound. You can continue to rework based on #16361 .
> 
> Thanks, I do really prefer fixing the file descriptor subsystem like @xiaoxiang781216 suggested in #16326 . It is better to do a comprehensive fix that as far as I can see, will solve most of your problems as well.

@pussuw @jlaitine 
Actually, this current PR doesn't just address the issue with fdcheck. 
Firstly, optimizing the file descriptors in epoll to be managed as files is quite reasonable, because fds shouldn't exist in the kernel in their raw form; they should be converted into `file structures` or the subsequent `struct fd` representation. https://github.com/apache/nuttx/pull/16431/commits/4629fc52164b8607b8c5343f1042425531d0df16
Secondly, we've removed poll_fdsetup to ensure that everyone uses interfaces with reference counting to access fds, which is much safer."https://github.com/apache/nuttx/pull/16431/commits/0cb3037484a143c3c72be4fe421a4fb8983e767a
The other two modifications are minor and only target the usage of file references.
author_association: CONTRIBUTOR
user: login: pussuw
created_at: 2025-05-23T12:19:03Z
body: > > > > > > Why is fdcheck implemented in the kernel, the kernel should not have to care about such things. Shouldn't it be implemented at the user interface i.e. libc ?
> > > > > 
> > > > > 
> > > > > @pussuw The implementation of fdcheck originally resides in libc https://github.com/apache/nuttx/blob/889a26db1d229ee462bc8ea075c2330dc5f79b94/libs/libc/misc/lib_fdcheck.c
> > > > > But we need to perform the conversion from file descriptor (fd) to file structure pointer (filep) within the kernel, as this conversion cannot be done in user space. And, the parameter for system calls is an fd, nothing else, and the filep is also a kernel data structure.
> > > > 
> > > > 
> > > > Yes the problem here is that the data is in filep. After #16361 you could do this magic in userspace.
> > > 
> > > 
> > > @pussuw #16361 would definitely be the best if we could cover this type of scenario. However, the changes in this current Pull Request (PR) are quite extensive, so it's unlikely to be merged into the mainline quickly. There's a possibility of stability issues, so we should ensure its reliability through some testing. Therefore, I will pick it up internally and conduct LTP (Linux Test Project) functional testing, 24-hour Monkey stress testing, as well as functional testing on multiple platforms, before approving it.
> > > Therefore, our priority is to ensure that the current design is sound. You can continue to rework based on #16361 .
> > 
> > 
> > Thanks, I do really prefer fixing the file descriptor subsystem like @xiaoxiang781216 suggested in #16326 . It is better to do a comprehensive fix that as far as I can see, will solve most of your problems as well.
> 
> @pussuw @jlaitine Actually, this current PR doesn't just address the issue with fdcheck. Firstly, optimizing the file descriptors in epoll to be managed as files is quite reasonable, because fds shouldn't exist in the kernel in their raw form; they should be converted into `file structures` or the subsequent `struct fd` representation. [4629fc5](https://github.com/apache/nuttx/commit/4629fc52164b8607b8c5343f1042425531d0df16) Secondly, we've removed poll_fdsetup to ensure that everyone uses interfaces with reference counting to access fds, which is much safer."[0cb3037](https://github.com/apache/nuttx/commit/0cb3037484a143c3c72be4fe421a4fb8983e767a) The other two modifications are minor and only target the usage of file references.

Yes, I agree with patches 1 and 4, this is why I did not comment on them, those could be merged with no disputes from my side
author_association: CONTRIBUTOR
user: login: xiaoxiang781216
created_at: 2025-05-24T14:07:03Z
body: @pussuw here is our plan:

1. Merge the file handle related fix found in our internal stress test before your file handle restructure patch
2. Resolve the conflict in your patch base on item 1 by @Donny9 
3. Run all internal function and stress tests to ensure all work as before by @Donny9 

Do you think this plan reasonable?
author_association: CONTRIBUTOR
user: login: xiaoxiang781216
created_at: 2025-05-24T14:11:09Z
body: > I completely agree with @pussuw here! There is nice cleanup available for the FDs in PR #16361, which fixes the operation to be acc. to the posix.
> 

Yes, all agree #16361 is the right direction to go.

> Please don't inject debug functionality which should NOT be inside kernel in there. Integrate first the #16361, and then implement the debug functionality in where it belongs. That is, in the kernel-userspace interface, not directly in the kernel.

The patchset doesn't change or add any new debugging functionality, but fix the nasty bug found by fdsan/fdcheck.
@jlaitine, we can improve the implementation detail of fdsan/fdcheck after merging this and #16361.
author_association: CONTRIBUTOR
user: login: Donny9
created_at: 2025-05-26T01:44:27Z
body: @pussuw please review. For the previous two patches, we made enhancements or implementations based on your PR.
author_association: CONTRIBUTOR
user: login: pussuw
created_at: 2025-05-26T09:29:18Z
body: > @pussuw here is our plan:
> 
> 1. Merge the file handle related fix found in our internal stress test before your file handle restructure patch
> 2. Resolve the conflict in your patch base on item 1 by @Donny9
> 3. Run all internal function and stress tests to ensure all work as before by @Donny9
> 
> Do you think this plan reasonable?

Yes, thank you. I'm OoO until tomorrow so don't have time to read github much until then.
author_association: CONTRIBUTOR
user: login: pussuw
created_at: 2025-05-26T09:29:54Z
body: > @pussuw please review. For the previous two patches, we made enhancements or implementations based on your PR.

Sorry, it will go to tomorrow
author_association: CONTRIBUTOR