Repository: bazelbuild/bazel
Issue: Insufficient system resources exist to complete the requsted service.
Labels: type: bug
P2
area-Windows
team-OSS

### Description of the problem / feature request:
Bazel installed by scoop or downlaoded binary crashes on run.
Prerequisites + curl + versions/python27 installed.

```
PS H:\> bazel --version
bazel 0.27.0
PS H:\> bazel help
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/388981886003af3376cd5aa448ae98ff/_embedded_binaries/embe
dded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_
bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDu
mpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the reque
sted service.

PS H:\> bazel
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/388981886003af3376cd5aa448ae98ff/_embedded_binaries/embe
dded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_
bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDu
mpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the reque
sted service.

PS H:\>
```
### Bugs: what's the simplest, easiest way to reproduce this bug? Please provide a minimal example if possible.

```
bazel
```

### What operating system are you running Bazel on?

```
OS Name:                   Microsoft Windows 7 Enterprise
OS Version:                6.1.7601 Service Pack 1 Build 7601
```
```
Processor(s):              1 Processor(s) Installed.
                           [01]: Intel64 Family 6 Model 58 Stepping 9 GenuineIntel ~3401 Mhz
```
```
Total Physical Memory:     32,655 MB
Available Physical Memory: 13,288 MB
Virtual Memory: Max Size:  32,653 MB
Virtual Memory: Available: 12,946 MB
Virtual Memory: In Use:    19,707 MB
```

### What's the output of `bazel info release`?

```
PS H:\> bazel info release
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/388981886003af3376cd5aa448ae98ff/_embedded_binaries/embe
dded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_
bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDu
mpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the reque
sted service.
```

###  Have you found anything relevant by searching the web?

https://stackoverflow.com/questions/56564539/how-do-i-give-bazel-more-resources

Temporary solution:
Install **bazel@0.25.0**, 0.26.0 is not working as well.

Issue comments:
/cc @meisterT 
Can you try with an explicit argument for `-Xmx`, e.g.
`bazel --host_jvm_args=-Xmx4g info`?
Great idea! I didn't think about it. Unfortunately I will test it on Monday.
Unfortunately
```bash
PS H:\> bazel --host_jvm_args=-Xmx4g info
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/388981886003af3376cd5aa448ae98ff/_embedded_binaries/embe
dded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_
bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDu
mpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the reque
sted service.
```

```bash
krzyskar@COMPUTERNAME MINGW64 /c/projects
$ bazel --host_jvm_args=-Xmx4g info
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/388981886003af3376cd5aa448ae98ff/_embedded_binaries/embedded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the requested service.
```

I tried also setting the initial memory allocation pool without luck

```bash
PS H:\> bazel --host_jvm_args=-Xms2G --host_jvm_args=-Xmx16G info
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/388981886003af3376cd5aa448ae98ff/_embedded_binaries/embe
dded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_
bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDu
mpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the reque
sted service.
```

More info:
```bash
PS H:\> C:\Users\krzyskar\_bazel_krzyskar\install\388981886003af3376cd5aa448ae98ff\_embedded_binaries\embedded_tools\jdk
\bin\java.exe -version
openjdk version "11.0.2" 2019-01-15 LTS
OpenJDK Runtime Environment Zulu11.29+3-CA (build 11.0.2+7-LTS)
OpenJDK 64-Bit Server VM Zulu11.29+3-CA (build 11.0.2+7-LTS, mixed mode)
```
Update
```
PS H:\> bazel --version
bazel 0.27.1
PS H:\> bazel info
FATAL: ExecuteProgram(C:\Users\krzyskar/_bazel_krzyskar/install/9470fb600225157f1e34c45c6d4dc834/_embedded_binaries/embe
dded_tools/jdk/bin/java.exe) failed: ERROR: src/main/native/windows/process.cc(184): CreateProcessW("C:\Users\krzyskar\_
bazel_krzyskar\install\9470fb600225157f1e34c45c6d4dc834\_embedded_binaries\embedded_tools\jdk\bin\java.exe"  -XX:+HeapDu
mpOnOutOfMemoryError -XX:HeapDumpPath=c:\\\users\\\krzy(...)): Insufficient system resources exist to complete the reque
sted service.
```
0.27.1 is not working as well.

Is it possible to use "external" JDK?
You can point Bazel to a different JDK with `--server_javabase` (https://docs.bazel.build/versions/master/command-line-reference.html#flag--server_javabase).
Is that message actually coming from the JVM or from Windows? cc @meteorcloudy in case he has further ideas.
Thank you @meisterT . I should check out the doc first.
```
PS H:\> echo $env:JAVA_HOME
C:\Users\krzyskar\scoop\apps\openjdk\current

PS H:\> Invoke-Expression $($env:JAVA_HOME + "\bin\java -version")
openjdk version "12.0.1" 2019-04-16
OpenJDK Runtime Environment (build 12.0.1+12)
OpenJDK 64-Bit Server VM (build 12.0.1+12, mixed mode, sharing)

PS H:\> bazel --server_javabase $env:JAVA_HOME
FATAL: ExecuteProgram(c:\users\krzyskar\scoop\apps\openjdk\current/bin/java.exe) failed: ERROR: src/main/native/windows/
process.cc(184): CreateProcessW("c:\users\krzyskar\scoop\apps\openjdk\current\bin\java.exe"  -XX:+HeapDumpOnOutOfMemoryE
rror -XX:HeapDumpPath=c:\\\users\\\krzyskar\\\_bazel_krzyskar\\\u5mzpase -XX:+UseParallelOldGC --add-opens=(...)): Insuf
ficient system resources exist to complete the requested service.
```

> Is that message actually coming from the JVM or from Windows?
How to differentiate that?

Interesting thing.
`bazel` fails on PS, cmd, ConEmu with Git Bash but works on Git Bash
```
krzyskar@COMPUTERNAME MINGW64 ~
$ bazel
WARNING: --batch mode is deprecated. Please instead explicitly shut down your Bazel server using the command "bazel shutdown".
                                                          [bazel release 0.27.1]
Usage: bazel <command> <options> ...
```
To my own surprise, I can repro this!

1. Created a GCP VM with Windows Server 2008, 1 CPU + 32GB RAM
2. Installed Bazel 0.27.1 through Chocolatey
3. Installed Zulu JDK 8, 64bit
4. Installed VC redist DLLs
5. Created a hello world `java_binary`
6. `bazel run //:hello` fails the same way @KrzysztofKarol described

I'll try to investigate.
More info:
- error is not Java specific: `bazel run` fails also for py_binary
- also not Bash specific: `bazel run` runs all binaries through bash.exe by default, but it also fails with `--incompatible_windows_bashless_run_command`, i.e. when not running through Bash
- there are enough resources: using any of the binaries in genrule.tools and building the genrule works

CreateProcess fails with ERROR_NO_SYSTEM_RESOURCES.

And https://bugzilla.mozilla.org/show_bug.cgi?id=1460995 seems very relevant: 
> CreateProcess fails with ERROR_NO_SYSTEM_RESOURCES on Windows 7. It looks like the reason why is because PROC_THREAD_ATTRIBUTE_HANDLE_LIST doesn't like console handles.
>
> We can fix this by only setting that policy (and its associated inheritable handles) when there is one or more standard handles that are *not* console handles.
>
> Note that if all handles are console handles, we don't need to pass STARTF_USESTDHANDLES either, since they'll automatically be set to the current console.
And that is indeed the problem!

Here:
https://github.com/bazelbuild/bazel/blob/6d0b14b95a71175362030b4811ca74512b00a890/src/main/native/windows/process.cc#L166-L187 when I don't use the attribute list, remove `EXTENDED_STARTUPINFO_PRESENT`, and use a `STARTUPINFOW` instead of `STARTUPINFOEXW` and let the subprocess inherit none of the handles, `bazel run` works. (And also prints to the console.)
Fixed by https://github.com/bazelbuild/bazel/pull/8793

Pull request review comments:
Although looks like this solves the problem, but we also need to make sure this doesn't leak any file handles other than `stdin`, `stdout` and `stderr` into the subprocess. 
I verified that it doesn't:
- created a program that uses the exact same code to create a subprocess
- the program also opens a file before creating the process
- i observed the subprocess' handles in Process Explorer.

With the current code, the subprocess does NOT inherit the file handle, even though I marked the handle inheritable.

If I change the code and pass the file handle as the stdout for the subprocess, then the handle is inherited and shown by process explorer.
Nice, thanks for the verifying!!

Patch that resolved the issue:
diff --git a/src/main/native/windows/process.cc b/src/main/native/windows/process.cc
index 05bb0f40385498..f00b76691f8578 100644
--- a/src/main/native/windows/process.cc
+++ b/src/main/native/windows/process.cc
@@ -50,14 +50,18 @@ static bool DupeHandle(HANDLE h, AutoHandle* out, std::wstring* error) {
 bool WaitableProcess::Create(const std::wstring& argv0,
                              const std::wstring& argv_rest, void* env,
                              const std::wstring& wcwd, std::wstring* error) {
-  AutoHandle dup_in, dup_out, dup_err;
-  if (!DupeHandle(GetStdHandle(STD_INPUT_HANDLE), &dup_in, error) ||
-      !DupeHandle(GetStdHandle(STD_OUTPUT_HANDLE), &dup_out, error) ||
-      !DupeHandle(GetStdHandle(STD_ERROR_HANDLE), &dup_err, error)) {
-    return false;
-  }
-  return Create(argv0, argv_rest, env, wcwd, dup_in, dup_out, dup_err, nullptr,
-                true, error);
+  // On Windows 7, the subprocess cannot inherit console handles via the
+  // attribute list (CreateProcessW fails with ERROR_NO_SYSTEM_RESOURCES).
+  // If we pass only invalid handles here, the Create method creates an
+  // AutoAttributeList with 0 handles and initializes the STARTUPINFOEXW as
+  // empty and disallowing handle inheritance. At the same time, CreateProcessW
+  // specifies neither CREATE_NEW_CONSOLE nor DETACHED_PROCESS, so the
+  // subprocess *does* inherit the console, which is exactly what we want, and
+  // it works on all supported Windows versions.
+  // Fixes https://github.com/bazelbuild/bazel/issues/8676
+  return Create(argv0, argv_rest, env, wcwd, INVALID_HANDLE_VALUE,
+                INVALID_HANDLE_VALUE, INVALID_HANDLE_VALUE, nullptr, true,
+                error);
 }
 
 bool WaitableProcess::Create(const std::wstring& argv0,
@@ -150,8 +154,8 @@ bool WaitableProcess::Create(const std::wstring& argv0,
   static constexpr size_t kMaxCmdline = 32767;
 
   std::wstring cmd_sample = mutable_commandline.get();
-  if (cmd_sample.size() > 200) {
-    cmd_sample = cmd_sample.substr(0, 195) + L"(...)";
+  if (cmd_sample.size() > 500) {
+    cmd_sample = cmd_sample.substr(0, 495) + L"(...)";
   }
   if (wcsnlen_s(mutable_commandline.get(), kMaxCmdline) == kMaxCmdline) {
     std::wstringstream error_msg;
@@ -181,8 +185,18 @@ bool WaitableProcess::Create(const std::wstring& argv0,
           /* lpStartupInfo */ &info.StartupInfo,
           /* lpProcessInformation */ &process_info)) {
     DWORD err = GetLastError();
+
+    std::wstring errmsg;
+    if (err == ERROR_NO_SYSTEM_RESOURCES && !IsWindows8OrGreater() &&
+        attr_list->HasConsoleHandle()) {
+      errmsg = L"Unrecoverable error: host OS is Windows 7 and subprocess"
+        " got an inheritable console handle";
+    } else {
+      errmsg = GetLastErrorString(err) + L" (error: " + ToString(err) + L")";
+    }
+
     *error = MakeErrorMessage(WSTR(__FILE__), __LINE__, L"CreateProcessW",
-                              cmd_sample, err);
+                              cmd_sample, errmsg);
     return false;
   }
 
diff --git a/src/main/native/windows/util.h b/src/main/native/windows/util.h
index 16357a1dc8c6dd..67eb84471fe523 100644
--- a/src/main/native/windows/util.h
+++ b/src/main/native/windows/util.h
@@ -81,6 +81,8 @@ class AutoAttributeList {
 
   void InitStartupInfoExW(STARTUPINFOEXW* startup_info) const;
 
+  bool HasConsoleHandle() const { return handles_.HasConsoleHandle(); }
+
  private:
   class StdHandles {
    public:
@@ -92,6 +94,15 @@ class AutoAttributeList {
     HANDLE StdOut() const { return stdout_h_; }
     HANDLE StdErr() const { return stderr_h_; }
 
+    bool HasConsoleHandle() const {
+      for (size_t i = 0; i < valid_handles_; ++i) {
+        if (GetFileType(valid_handle_array_[i]) == FILE_TYPE_CHAR) {
+          return true;
+        }
+      }
+      return false;
+    }
+
    private:
     size_t valid_handles_;
     HANDLE valid_handle_array_[3];


Test patch: