Repository: huggingface/evaluate
Issue: os.unlink(entry.name, dir_fd=topfd)  TypeError: 'NoneType' object is not callable
Root cause: Validation Gap

When I use code_eval.py to do evaluation I found the following problem.

```
Traceback (most recent call last):
  File "/opt/conda/lib/python3.10/weakref.py", line 667, in _exitfunc
    f()
  File "/opt/conda/lib/python3.10/weakref.py", line 591, in __call__
    return info.func(*info.args, **(info.kwargs or {}))
  File "/opt/conda/lib/python3.10/tempfile.py", line 858, in _cleanup
    cls._rmtree(name, ignore_errors=ignore_errors)
  File "/opt/conda/lib/python3.10/tempfile.py", line 854, in _rmtree
    _shutil.rmtree(name, onerror=onerror)
  File "/opt/conda/lib/python3.10/shutil.py", line 728, in rmtree
    _rmtree_safe_fd(fd, path, onerror)
  File "/opt/conda/lib/python3.10/shutil.py", line 682, in _rmtree_safe_fd
    os.unlink(entry.name, dir_fd=topfd)
TypeError: 'NoneType' object is not callable
```

I found /root/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--code_eval/78d307ea938083398db7d9815f03ed661e9c15f60d77880ce007a8a02648f176/execute.py

```
def reliability_guard(maximum_memory_bytes=None):
    """
    This disables various destructive functions and prevents the generated code
    from interfering with the test (e.g. fork bomb, killing other processes,
    removing filesystem files, etc.)

    WARNING
    This function is NOT a security sandbox. Untrusted code, including, model-
    generated code, should not be blindly executed outside of one. See the
    Codex paper for more information about OpenAI's code sandbox, and proceed
    with caution.
    """
...
os.replace = None
os.unlink = None
os.fchmod = None
...
```
So, the os.unlink should be None. But in the 
metrics/code_eval/execute.py
```
def unsafe_execute(check_program, result, timeout):

    with create_tempdir():

        # These system calls are needed when cleaning up tempdir.
        import os
        import shutil

        rmtree = shutil.rmtree
        rmdir = os.rmdir
        chdir = os.chdir
```
then, we found it called
```
@contextlib.contextmanager
def create_tempdir():
    with tempfile.TemporaryDirectory() as dirname:
        with chdir(dirname):
            yield dirname
```
For tempfile.TemporaryDirectory(), when exit this function, it will call 
--> def rmtree(path, ignore_errors=False, onerror=None):
  --> _rmtree_safe_fd(fd, path, onerror
    --> 
```
def _rmtree_safe_fd(topfd, path, onerror):
    try:
        with os.scandir(topfd) as scandir_it:
            entries = list(scandir_it)
    except OSError as err:
        err.filename = path
        onerror(os.scandir, path, sys.exc_info())
        return
    for entry in entries:
        fullname = os.path.join(path, entry.name)
        try:
            is_dir = entry.is_dir(follow_symlinks=False)
        except OSError:
            is_dir = False
        else:
            if is_dir:
                try:
                    orig_st = entry.stat(follow_symlinks=False)
                    is_dir = stat.S_ISDIR(orig_st.st_mode)
                except OSError:
                    onerror(os.lstat, fullname, sys.exc_info())
                    continue
        if is_dir:
            try:
                dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
                dirfd_closed = False
            except OSError:
                onerror(os.open, fullname, sys.exc_info())
            else:
                try:
                    if os.path.samestat(orig_st, os.fstat(dirfd)):
                        _rmtree_safe_fd(dirfd, fullname, onerror)
                        try:
                            os.close(dirfd)
                            dirfd_closed = True
                            os.rmdir(entry.name, dir_fd=topfd)
                        except OSError:
                            onerror(os.rmdir, fullname, sys.exc_info())
                    else:
                        try:
                            # This can only happen if someone replaces
                            # a directory with a symlink after the call to
                            # os.scandir or stat.S_ISDIR above.
                            raise OSError("Cannot call rmtree on a symbolic "
                                          "link")
                        except OSError:
                            onerror(os.path.islink, fullname, sys.exc_info())
                finally:
                    if not dirfd_closed:
                        os.close(dirfd)
        else:
            try:
                os.unlink(entry.name, dir_fd=topfd) # NOTICE!!! It will run this position with os.unlink is None and lead an error mentioned above.
            except OSError:
                onerror(os.unlink, fullname, sys.exc_info())
```

My modification is :
```
--- a/metrics/code_eval/execute.py
+++ b/metrics/code_eval/execute.py
@@ -64,6 +64,7 @@ def unsafe_execute(check_program, result, timeout):
         rmtree = shutil.rmtree
         rmdir = os.rmdir
         chdir = os.chdir
+        osunlink = os.unlink

         # Disable functionalities that can make destructive changes to the test.
         reliability_guard()
@@ -84,6 +85,7 @@ def unsafe_execute(check_program, result, timeout):
         shutil.rmtree = rmtree
         os.rmdir = rmdir
         os.chdir = chdir
+        os.unlink = osunlink
```
Is there any better method?

Thanks