CVE-2021-4213: Missing Release of Memory after Effective Lifetime (CWE-401)
value: A flaw was found in JSS, where it did not properly free up all memory. Over time, the wasted memory builds up in the server memory, saturating the server’s RAM. This flaw allows an attacker to force the invocation of an out-of-memory process, causing a denial of service.
CWE description: The product does not sufficiently track and release allocated memory after it has been used, which slowly consumes remaining memory.
CVSS v3 base score: 7.5
Published: 2022-08-24T16:15Z
Repository: https://github.com/dogtagpki/jss
Language: Java
Files changed: src/main/java/org/mozilla/jss/nss/SSLFDProxy.java, src/main/java/org/mozilla/jss/ssl/javax/JSSEngineReferenceImpl.java, src/main/java/org/mozilla/jss/util/GlobalRefProxy.java
Commit date: 2022-02-11 20:43:58 +0000
Commit message: Fix memory leak on each TLS connection

Each TLS connection is leaking a bunch of data that isn't in the heap
and so after 25k requests Tomcat uses about 2.5GB resident memory.

There are large number of relationships that point at each other and we
need to break the cycle so JSSEngineReferenceImpl's finalizer can run
and clear all the native resources these point at.

The lowest impact place to break the cycle was at SSLAlertEvent.engine.
This relationship doesn't seem to be used anywhere.  Once the cycle is
broken, JSSEngineReferenceImpl can be garbage collected and the
finalizer can run.

Signed-off-by: Chris Kelley <ckelley@redhat.com>
Diff: diff --git a/src/main/java/org/mozilla/jss/nss/SSLFDProxy.java b/src/main/java/org/mozilla/jss/nss/SSLFDProxy.java
index fc3af3e1..7cbbe2dd 100644
--- a/src/main/java/org/mozilla/jss/nss/SSLFDProxy.java
+++ b/src/main/java/org/mozilla/jss/nss/SSLFDProxy.java
@@ -1,8 +1,7 @@
 package org.mozilla.jss.nss;
 
-import java.lang.IllegalArgumentException;
 import java.util.ArrayList;
 
 import org.mozilla.jss.crypto.X509Certificate;
 import org.mozilla.jss.pkcs11.PK11Cert;
 import org.mozilla.jss.ssl.SSLAlertEvent;
@@ -40,17 +39,18 @@ public class SSLFDProxy extends PRFDProxy {
         clientCert = (PK11Cert)cert;
     }
 
     @Override
     protected synchronized void releaseNativeResources() throws Exception {
-        synchronized (globalRef) {
-            if (globalRef != null) {
-                try {
-                    globalRef.close();
-                } finally {
-                    globalRef = null;
-                }
+
+        super.releaseNativeResources();
+
+        if (globalRef != null) {
+            try {
+                globalRef.close();
+            } finally {
+                globalRef = null;
             }
         }
     }
 
     public int invokeCertAuthHandler() {
diff --git a/src/main/java/org/mozilla/jss/ssl/javax/JSSEngineReferenceImpl.java b/src/main/java/org/mozilla/jss/ssl/javax/JSSEngineReferenceImpl.java
index 74b85925..93b290c3 100644
--- a/src/main/java/org/mozilla/jss/ssl/javax/JSSEngineReferenceImpl.java
+++ b/src/main/java/org/mozilla/jss/ssl/javax/JSSEngineReferenceImpl.java
@@ -2,21 +2,43 @@ package org.mozilla.jss.ssl.javax;
 
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.ServerSocket;
 import java.net.Socket;
-import java.nio.channels.WritableByteChannel;
+import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
 import java.security.PublicKey;
-import java.nio.ByteBuffer;
-
-import javax.net.ssl.*;
 
-import org.mozilla.jss.nss.*;
-import org.mozilla.jss.pkcs11.*;
-import org.mozilla.jss.provider.javax.crypto.*;
-import org.mozilla.jss.ssl.*;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.X509ExtendedTrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.mozilla.jss.nss.BadCertHandler;
+import org.mozilla.jss.nss.Buffer;
+import org.mozilla.jss.nss.BufferProxy;
+import org.mozilla.jss.nss.Cert;
+import org.mozilla.jss.nss.CertAuthHandler;
+import org.mozilla.jss.nss.PR;
+import org.mozilla.jss.nss.PRErrors;
+import org.mozilla.jss.nss.PRFDProxy;
+import org.mozilla.jss.nss.SSL;
+import org.mozilla.jss.nss.SSLErrors;
+import org.mozilla.jss.nss.SSLFDProxy;
+import org.mozilla.jss.nss.SSLPreliminaryChannelInfo;
+import org.mozilla.jss.nss.SecurityStatusResult;
+import org.mozilla.jss.pkcs11.PK11Cert;
+import org.mozilla.jss.provider.javax.crypto.JSSNativeTrustManager;
+import org.mozilla.jss.ssl.SSLAlertDescription;
+import org.mozilla.jss.ssl.SSLAlertEvent;
+import org.mozilla.jss.ssl.SSLAlertLevel;
+import org.mozilla.jss.ssl.SSLCipher;
+import org.mozilla.jss.ssl.SSLHandshakeCompletedEvent;
+import org.mozilla.jss.ssl.SSLVersion;
+import org.mozilla.jss.ssl.SSLVersionRange;
 
 /**
  * The reference JSSEngine implementation.
  *
  * This JSSEngine implementation is a reference for future JSSEngine
@@ -1649,14 +1671,14 @@ public class JSSEngineReferenceImpl extends JSSEngine {
 
     private void cleanupSSLFD() {
         if (!closed_fd && ssl_fd != null) {
             try {
                 SSL.RemoveCallbacks(ssl_fd);
-                PR.Close(ssl_fd);
                 ssl_fd.close();
+                ssl_fd = null;
             } catch (Exception e) {
-                debug("Got exception trying to cleanup SSLFD: " + e.getMessage());
+                logger.error("Got exception trying to cleanup SSLFD", e);
             } finally {
                 closed_fd = true;
             }
         }
 
@@ -1669,10 +1691,19 @@ public class JSSEngineReferenceImpl extends JSSEngine {
             Buffer.Free(write_buf);
             write_buf = null;
         }
     }
 
+    // During testing with Tomcat 8.5, most instances did not call
+    // cleanup, so all the JNI resources end up getting leaked: ssl_fd
+    // (and its global ref), read_buf, and write_buf.
+    @Override
+    protected void finalize() {
+        cleanup();
+    }
+
+
     private class CertValidationTask extends CertAuthHandler {
         public CertValidationTask(SSLFDProxy fd) {
             super(fd);
         }
 
diff --git a/src/main/java/org/mozilla/jss/util/GlobalRefProxy.java b/src/main/java/org/mozilla/jss/util/GlobalRefProxy.java
index 8bdbf6f6..a18ace2d 100644
--- a/src/main/java/org/mozilla/jss/util/GlobalRefProxy.java
+++ b/src/main/java/org/mozilla/jss/util/GlobalRefProxy.java
@@ -10,9 +10,7 @@ public class GlobalRefProxy extends NativeProxy {
     }
 
     private static native byte[] refOf(Object target);
 
     @Override
-    protected synchronized void releaseNativeResources() {
-        clear();
-    }
+    protected native void releaseNativeResources();
 }

Vulnerable code: import java.lang.IllegalArgumentException;
        synchronized (globalRef) {
            if (globalRef != null) {
                try {
                    globalRef.close();
                } finally {
                    globalRef = null;
                }
import java.nio.channels.WritableByteChannel;
import java.nio.ByteBuffer;

import javax.net.ssl.*;
import org.mozilla.jss.nss.*;
import org.mozilla.jss.pkcs11.*;
import org.mozilla.jss.provider.javax.crypto.*;
import org.mozilla.jss.ssl.*;
                PR.Close(ssl_fd);
                debug("Got exception trying to cleanup SSLFD: " + e.getMessage());
    protected synchronized void releaseNativeResources() {
        clear();
    }
Fixed code: 
        super.releaseNativeResources();

        if (globalRef != null) {
            try {
                globalRef.close();
            } finally {
                globalRef = null;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import javax.net.ssl.SSLEngineResult;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.X509ExtendedTrustManager;
import javax.net.ssl.X509TrustManager;

import org.mozilla.jss.nss.BadCertHandler;
import org.mozilla.jss.nss.Buffer;
import org.mozilla.jss.nss.BufferProxy;
import org.mozilla.jss.nss.Cert;
import org.mozilla.jss.nss.CertAuthHandler;
import org.mozilla.jss.nss.PR;
import org.mozilla.jss.nss.PRErrors;
import org.mozilla.jss.nss.PRFDProxy;
import org.mozilla.jss.nss.SSL;
import org.mozilla.jss.nss.SSLErrors;
import org.mozilla.jss.nss.SSLFDProxy;
import org.mozilla.jss.nss.SSLPreliminaryChannelInfo;
import org.mozilla.jss.nss.SecurityStatusResult;
import org.mozilla.jss.pkcs11.PK11Cert;
import org.mozilla.jss.provider.javax.crypto.JSSNativeTrustManager;
import org.mozilla.jss.ssl.SSLAlertDescription;
import org.mozilla.jss.ssl.SSLAlertEvent;
import org.mozilla.jss.ssl.SSLAlertLevel;
import org.mozilla.jss.ssl.SSLCipher;
import org.mozilla.jss.ssl.SSLHandshakeCompletedEvent;
import org.mozilla.jss.ssl.SSLVersion;
import org.mozilla.jss.ssl.SSLVersionRange;
                ssl_fd = null;
                logger.error("Got exception trying to cleanup SSLFD", e);
    // During testing with Tomcat 8.5, most instances did not call
    // cleanup, so all the JNI resources end up getting leaked: ssl_fd
    // (and its global ref), read_buf, and write_buf.
    @Override
    protected void finalize() {
        cleanup();
    }


    protected native void releaseNativeResources();