CVE-2020-36634: Improper Neutralization of Input During Web Page Generation ('Cross-site Scripting') (CWE-79)
value: A vulnerability classified as problematic has been found in Indeed Engineering util up to 1.0.33. Affected is the function visit/appendTo of the file varexport/src/main/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServlet.java. The manipulation leads to cross site scripting. It is possible to launch the attack remotely. Upgrading to version 1.0.34 is able to address this issue. The name of the patch is c0952a9db51a880e9544d9fac2a2218a6bfc9c63. It is recommended to upgrade the affected component. VDB-216882 is the identifier assigned to this vulnerability.
CWE description: The product does not neutralize or incorrectly neutralizes user-controllable input before it is placed in output that is used as a web page that is served to other users.
CVSS v3 base score: 5.4
Published: 2022-12-27T13:15Z
Repository: https://github.com/indeedeng/util
Language: Java
Files changed: varexport/src/main/java/com/indeed/util/varexport/Variable.java, varexport/src/main/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServlet.java, varexport/src/test/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServletTest.java
Commit date: 2020-07-20 14:49:15 +0900
Commit message: COMMON-3969: Explicit alphanumeric n-gram indexes

Explicitly build alphanumeric n-gram indexes to avoid XSS due
to unescaped characters in the rendered n-gram indexes JSON

Diff: diff --git a/varexport/src/main/java/com/indeed/util/varexport/Variable.java b/varexport/src/main/java/com/indeed/util/varexport/Variable.java
index 061ed38..9b51748 100644
--- a/varexport/src/main/java/com/indeed/util/varexport/Variable.java
+++ b/varexport/src/main/java/com/indeed/util/varexport/Variable.java
@@ -49,10 +49,15 @@ public abstract class Variable<T> {
 
     public String getName() {
         return name;
     }
 
+    /**
+     * @deprecated do not use this.
+     * Please implement your own tokenization of variable names, if needed.
+     */
+    @Deprecated
     public String[] getIndexableNames() {
         return indexableNames;
     }
 
     public String getDoc() {
diff --git a/varexport/src/main/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServlet.java b/varexport/src/main/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServlet.java
index da9a119..cfb4ab2 100644
--- a/varexport/src/main/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServlet.java
+++ b/varexport/src/main/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServlet.java
@@ -2,20 +2,25 @@
 package com.indeed.util.varexport.servlet;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.TreeMultimap;
 import com.indeed.util.varexport.VarExporter;
 import com.indeed.util.varexport.Variable;
 import com.indeed.util.varexport.VariableHost;
 import com.indeed.util.varexport.VariableVisitor;
 import freemarker.template.Configuration;
 import freemarker.template.DefaultObjectWrapper;
 import freemarker.template.Template;
 import org.apache.log4j.Logger;
 
+import javax.annotation.Nullable;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
@@ -23,11 +28,22 @@ import javax.servlet.http.HttpServletResponse;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 /**
  * Servlet for displaying variables exported by {@link VarExporter}.
  * Will escape values for compatibility with loading into {@link Properties}.
  *
@@ -285,34 +301,38 @@ public class ViewExportedVariablesServlet extends HttpServlet {
                 }
             }
         }
         root.put("vars", varList);
         if (withIndex) {
-            final String varsIndex = buildIndex(varList);
-            root.put("varsIndex", varsIndex);
+            root.put("varsIndex", alphanumericNGramIndexesJSON(varList));
         }
 
         try {
             template.process(root, out);
         } catch (Exception e) {
             throw new IOException("template failure", e);
         }
     }
 
-    private String buildIndex(final List<Variable> varList) {
-        final SetMultimap<String, Integer> uniGram = buildNGramIndex(varList, 1);
-        final SetMultimap<String, Integer> biGram = buildNGramIndex(varList, 2);
-        final SetMultimap<String, Integer> triGram = buildNGramIndex(varList, 3);
+    private String alphanumericNGramIndexesJSON(final List<Variable> varList) {
+        // The JSON we build here is valid as we only use alphanumeric n-grams
+
         final StringBuilder json = new StringBuilder();
+
         json.append('{').append('\n');
-        json.append("\"uniGram\":").append('\n');
-        appendTo(json, uniGram).append(',').append('\n');
+
+        json.append("\n\"uniGram\":").append('\n');
+        appendTo(json, buildAlphanumericNGramIndex(varList, 1)).append(',').append('\n');
+
         json.append("\"biGram\":").append('\n');
-        appendTo(json, biGram).append(',').append('\n');;
+        appendTo(json, buildAlphanumericNGramIndex(varList, 2)).append(',').append('\n');;
+
         json.append("\"triGram\":").append('\n');
-        appendTo(json, triGram);
+        appendTo(json, buildAlphanumericNGramIndex(varList, 3));
+
         json.append('}');
+
         return json.toString();
     }
 
     private <K> StringBuilder appendTo(final StringBuilder json, final SetMultimap<K, Integer> map) {
         json.append('{').append('\n');
@@ -331,23 +351,36 @@ public class ViewExportedVariablesServlet extends HttpServlet {
         }
         json.append('\n').append('}');
         return json;
     }
 
-    private SetMultimap<String, Integer> buildNGramIndex(final List<Variable> varList, final int n) {
-        final TreeMultimap<String, Integer> uniGram = TreeMultimap.create();
-        for (int index = 0; index < varList.size(); index++) {
-            final Variable var = varList.get(index);
-            final String[] indexableNames = var.getIndexableNames();
-            for (final String indexableName : indexableNames) {
-                for (int i = 0; i < indexableName.length() - n + 1; i++) {
-                    final String key = indexableName.substring(i, i + n);
-                    uniGram.put(key, index);
-                }
-            }
+    @VisibleForTesting
+    static SetMultimap<String, Integer> buildAlphanumericNGramIndex(final List<Variable> varList, final int n) {
+        final TreeMultimap<String, Integer> nGramIndex = TreeMultimap.create();
+        IntStream
+                .range(0, varList.size())
+                .forEach(i ->
+                        alphanumericNGrams(varList.get(i).getName(), n)
+                                .forEach(ngram -> nGramIndex.put(ngram, i))
+                );
+        return nGramIndex;
+    }
+
+    private static final Pattern NON_ALPHANUMERIC = Pattern.compile("[^a-z0-9]+");
+
+    @VisibleForTesting
+    static Stream<String> alphanumericNGrams(@Nullable final String in, final int n) {
+        if (in == null) {
+            return Stream.empty();
         }
-        return uniGram;
+        return NON_ALPHANUMERIC
+                .splitAsStream(in.toLowerCase(Locale.US))
+                .flatMap(alnumSubString ->
+                        IntStream
+                                .rangeClosed(0, alnumSubString.length() - n)
+                                .mapToObj(i -> alnumSubString.substring(i, i + n))
+                );
     }
 
     private void addVariable(final Variable v, final List<Variable> out) {
         try {
             v.toString();
diff --git a/varexport/src/test/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServletTest.java b/varexport/src/test/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServletTest.java
index dabdc33..cba59ea 100644
--- a/varexport/src/test/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServletTest.java
+++ b/varexport/src/test/java/com/indeed/util/varexport/servlet/ViewExportedVariablesServletTest.java
@@ -1,9 +1,11 @@
 package com.indeed.util.varexport.servlet;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.TreeMultimap;
 import com.indeed.util.varexport.Export;
 import com.indeed.util.varexport.ManagedVariable;
 import com.indeed.util.varexport.VarExporter;
 import freemarker.template.Configuration;
 import freemarker.template.DefaultObjectWrapper;
@@ -17,17 +19,20 @@ import org.junit.Test;
 
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 
 /**
  * @author jack@indeed.com (Jack Humphrey)
@@ -162,10 +167,67 @@ public class ViewExportedVariablesServletTest {
         verify(response);
 
         assertFalse(bodyWriter.toString().contains(unescaped));
     }
 
+    @Test
+    public void buildAlphaNumericNGramIndex() {
+        final TreeMultimap<String, Integer> expectedTriGramIndex = TreeMultimap.create();
+        expectedTriGramIndex.put("abc", 0);
+        expectedTriGramIndex.put("bcd", 0);
+        expectedTriGramIndex.put("bcd", 1);
+        expectedTriGramIndex.put("zzz", 2);
+
+        final SetMultimap<String, Integer> actualTriGramIndex = ViewExportedVariablesServlet.buildAlphanumericNGramIndex(
+                Arrays.asList(
+                        ManagedVariable.builder().setName("abcd").build(),
+                        ManagedVariable.builder().setName("bcd").build(),
+                        ManagedVariable.builder().setName("zzz").build()
+                ),
+                3
+        );
+
+        assertEquals(expectedTriGramIndex, actualTriGramIndex);
+    }
+
+    @Test
+    public void alphanumericNGrams() {
+        assertEquals(0, ViewExportedVariablesServlet.alphanumericNGrams(null, 1).count());
+        assertEquals(0, ViewExportedVariablesServlet.alphanumericNGrams("zzz", 42).count());
+
+        assertEquals(
+                Arrays.asList("m", "1", "3", "b", "i", "t", "r", "i"),
+                ViewExportedVariablesServlet
+                        .alphanumericNGrams("m1-3-/bi<</tri", 1)
+                        .collect(Collectors.toList())
+        );
+
+        assertEquals(
+                Arrays.asList("m1", "bi", "tr", "ri"),
+                ViewExportedVariablesServlet
+                        .alphanumericNGrams("m1-3-/bi<</tri", 2)
+                        .collect(Collectors.toList())
+        );
+
+        assertEquals(
+                Arrays.asList(
+                        "m1x",
+                        "1x3",
+                        "x3d",
+                        "3dc",
+                        "dca",
+                        "cas",
+                        "ase",
+                        "tri",
+                        "tri"
+                ),
+                ViewExportedVariablesServlet
+                        .alphanumericNGrams("m1x3dCaSe-1-/bi<</tri?tri", 3)
+                        .collect(Collectors.toList())
+        );
+    }
+
     @Before
     public void setUp() throws Exception {
         VarExporter.resetGlobal();
     }
 

Vulnerable code: import com.google.common.collect.*;
import java.util.*;
            final String varsIndex = buildIndex(varList);
            root.put("varsIndex", varsIndex);
    private String buildIndex(final List<Variable> varList) {
        final SetMultimap<String, Integer> uniGram = buildNGramIndex(varList, 1);
        final SetMultimap<String, Integer> biGram = buildNGramIndex(varList, 2);
        final SetMultimap<String, Integer> triGram = buildNGramIndex(varList, 3);
        json.append("\"uniGram\":").append('\n');
        appendTo(json, uniGram).append(',').append('\n');
        appendTo(json, biGram).append(',').append('\n');;
        appendTo(json, triGram);
    private SetMultimap<String, Integer> buildNGramIndex(final List<Variable> varList, final int n) {
        final TreeMultimap<String, Integer> uniGram = TreeMultimap.create();
        for (int index = 0; index < varList.size(); index++) {
            final Variable var = varList.get(index);
            final String[] indexableNames = var.getIndexableNames();
            for (final String indexableName : indexableNames) {
                for (int i = 0; i < indexableName.length() - n + 1; i++) {
                    final String key = indexableName.substring(i, i + n);
                    uniGram.put(key, index);
                }
            }
        return uniGram;
Fixed code:     /**
     * @deprecated do not use this.
     * Please implement your own tokenization of variable names, if needed.
     */
    @Deprecated
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.TreeMultimap;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
import java.util.stream.Stream;
            root.put("varsIndex", alphanumericNGramIndexesJSON(varList));
    private String alphanumericNGramIndexesJSON(final List<Variable> varList) {
        // The JSON we build here is valid as we only use alphanumeric n-grams



        json.append("\n\"uniGram\":").append('\n');
        appendTo(json, buildAlphanumericNGramIndex(varList, 1)).append(',').append('\n');

        appendTo(json, buildAlphanumericNGramIndex(varList, 2)).append(',').append('\n');;

        appendTo(json, buildAlphanumericNGramIndex(varList, 3));


    @VisibleForTesting
    static SetMultimap<String, Integer> buildAlphanumericNGramIndex(final List<Variable> varList, final int n) {
        final TreeMultimap<String, Integer> nGramIndex = TreeMultimap.create();
        IntStream
                .range(0, varList.size())
                .forEach(i ->
                        alphanumericNGrams(varList.get(i).getName(), n)
                                .forEach(ngram -> nGramIndex.put(ngram, i))
                );
        return nGramIndex;
    }

    private static final Pattern NON_ALPHANUMERIC = Pattern.compile("[^a-z0-9]+");

    @VisibleForTesting
    static Stream<String> alphanumericNGrams(@Nullable final String in, final int n) {
        if (in == null) {
            return Stream.empty();
        return NON_ALPHANUMERIC
                .splitAsStream(in.toLowerCase(Locale.US))
                .flatMap(alnumSubString ->
                        IntStream
                                .rangeClosed(0, alnumSubString.length() - n)
                                .mapToObj(i -> alnumSubString.substring(i, i + n))
                );
import com.google.common.collect.SetMultimap;
import com.google.common.collect.TreeMultimap;
import java.util.Arrays;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
    @Test
    public void buildAlphaNumericNGramIndex() {
        final TreeMultimap<String, Integer> expectedTriGramIndex = TreeMultimap.create();
        expectedTriGramIndex.put("abc", 0);
        expectedTriGramIndex.put("bcd", 0);
        expectedTriGramIndex.put("bcd", 1);
        expectedTriGramIndex.put("zzz", 2);

        final SetMultimap<String, Integer> actualTriGramIndex = ViewExportedVariablesServlet.buildAlphanumericNGramIndex(
                Arrays.asList(
                        ManagedVariable.builder().setName("abcd").build(),
                        ManagedVariable.builder().setName("bcd").build(),
                        ManagedVariable.builder().setName("zzz").build()
                ),
                3
        );

        assertEquals(expectedTriGramIndex, actualTriGramIndex);
    }

    @Test
    public void alphanumericNGrams() {
        assertEquals(0, ViewExportedVariablesServlet.alphanumericNGrams(null, 1).count());
        assertEquals(0, ViewExportedVariablesServlet.alphanumericNGrams("zzz", 42).count());

        assertEquals(
                Arrays.asList("m", "1", "3", "b", "i", "t", "r", "i"),
                ViewExportedVariablesServlet
                        .alphanumericNGrams("m1-3-/bi<</tri", 1)
                        .collect(Collectors.toList())
        );

        assertEquals(
                Arrays.asList("m1", "bi", "tr", "ri"),
                ViewExportedVariablesServlet
                        .alphanumericNGrams("m1-3-/bi<</tri", 2)
                        .collect(Collectors.toList())
        );

        assertEquals(
                Arrays.asList(
                        "m1x",
                        "1x3",
                        "x3d",
                        "3dc",
                        "dca",
                        "cas",
                        "ase",
                        "tri",
                        "tri"
                ),
                ViewExportedVariablesServlet
                        .alphanumericNGrams("m1x3dCaSe-1-/bi<</tri?tri", 3)
                        .collect(Collectors.toList())
        );
    }