Repository: google/closure-compiler
Pull request #1253 (2015-11-16 09:09:34)

Support multiplexing files via stdin/out streams
As discusses last meeting, this is the initial work to support sending multiple input files to the compiler via stdin. This is needed to support stream based build systems like [Gulp](http://gulpjs.com/).

This currently uses a new boolean flag `--json_streams`. Ideally we'd avoid this, but I haven't come up with another good solution.

When the `--json_streams` flag is true, standard input is expected to be an array of json encoded files of the form:

```JSON
[
  {
    "src": "file source",
    "path": "/absolute/path/of/file.js",
    "source_map": "optional. source of input sourcemap"
  }
]
```

When the compiled source is written to standard output, it will also use the same format.

Currently this only supports single binaries. I'll be working to add modules shortly.

dbkaplun: +1 keep it up @ChadKillingsworth!
ChadKillingsworth: This is now ready for review and is needed for gulp/grunt plugins. See https://github.com/ChadKillingsworth/closure-compiler-npm

I'd like to have this merged before the next release.

Once any review items are addressed, I'll rebase and squash the commits to the top of the tree.
dimvar: OK. I can take a look tomorrow.
ChadKillingsworth: I've not tested this on a windows machine as well as in projects that make use of the `--module` flags. Found a couple of bugs, but changes have now been pushed.

Overall it's working very well.
dimvar: This will have to wait until Monday after all. Got caught up with other things.
ChadKillingsworth: No worries. I just got the windows box testing completed this afternoon. I'm testing it on projects that pipe all of closure-library in this way and it performs very well.
dimvar: Hey Chad, I left a few comments, but Github's bad workflow for code reviews makes this hard to read. Do you mind squashing the commits? Then I can use a tool to bring in the PR internally and use the nicer internal tool to look at the diffs.
ChadKillingsworth: Commits squashed. I have a full test suite for this at https://github.com/ChadKillingsworth/closure-compiler-npm/tree/master/test
ChadKillingsworth: FYI this fixes the last of the issues with #905 
dimvar: OK, I finished going through the whole thing. I'll take another look after you address these comments.
ChadKillingsworth: I think I've address all the review comments.
dimvar: Looks good. Only a couple minor comments.
dimvar: I'll wait for Ben to do the daily sync and then I'll merge this.

Patch:
diff --git a/contrib/externs/angular-1.4.js b/contrib/externs/angular-1.4.js
index a27e0827b..191c275d0 100644
--- a/contrib/externs/angular-1.4.js
+++ b/contrib/externs/angular-1.4.js
@@ -1528,7 +1528,7 @@ angular.$HttpProvider = function() {};
 angular.$HttpProvider.prototype.defaults;
 
 /**
- * @type {!Array.<string|function(...?): !angular.$http.Interceptor>}
+ * @type {!Array.<string|function(...*): !angular.$http.Interceptor>}
  */
 angular.$HttpProvider.prototype.interceptors;
 
diff --git a/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java b/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java
index 86386c166..d47e16f68 100644
--- a/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java
+++ b/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java
@@ -30,6 +30,9 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.io.ByteStreams;
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonWriter;
 import com.google.javascript.jscomp.CompilerOptions.TweakProcessing;
 import com.google.javascript.jscomp.deps.ClosureBundler;
 import com.google.javascript.jscomp.deps.SourceCodeEscapers;
@@ -45,6 +48,7 @@ import java.io.FileOutputStream;
 import java.io.Flushable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintStream;
@@ -172,9 +176,13 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
 
   private Map<String, String> parsedModuleWrappers = null;
 
+  private final Gson gson;
+
   static final String OUTPUT_MARKER = "%output%";
   private static final String OUTPUT_MARKER_JS_STRING = "%output|jsstring%";
 
+  private final List<JsonFileSpec> filesToStreamOut = new ArrayList<>();
+
   AbstractCommandLineRunner() {
     this(System.out, System.err);
   }
@@ -183,6 +191,7 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
     this.config = new CommandLineConfig();
     this.defaultJsOutput = Preconditions.checkNotNull(out);
     this.err = Preconditions.checkNotNull(err);
+    this.gson = new Gson();
   }
 
   /**
@@ -359,6 +368,8 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
 
     if (config.createSourceMap.length() > 0) {
       options.sourceMapOutputPath = config.createSourceMap;
+    } else if (config.useJsonStreams) {
+      options.sourceMapOutputPath = "%outname%";
     }
     options.sourceMapDetailLevel = config.sourceMapDetailLevel;
     options.sourceMapFormat = config.sourceMapFormat;
@@ -555,6 +566,19 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
     }
   }
 
+  public List<JsonFileSpec> parseJsonFilesFromInputStream() throws IOException {
+    List<JsonFileSpec> jsonFiles = new ArrayList<>();
+    JsonReader reader = new JsonReader(new InputStreamReader(System.in, inputCharset));
+    reader.beginArray();
+    while (reader.hasNext()) {
+      JsonFileSpec jsonFile = gson.fromJson(reader, JsonFileSpec.class);
+      jsonFiles.add(jsonFile);
+    }
+    reader.endArray();
+    reader.close();
+    return jsonFiles;
+  }
+
   /**
    * Creates inputs from a list of files.
    *
@@ -571,6 +595,21 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
     return createInputs(files, new ArrayList<String>() /* zips */, allowStdIn);
   }
 
+  /**
+   * Creates inputs from a list of source files and json files.
+   *
+   * Can be overridden by subclasses who want to pull files from different
+   * places.
+   *
+   * @param files A list of filenames.
+   * @param jsonFiles A list of json encoded files.
+   * @return An array of inputs
+   */
+  protected List<SourceFile> createInputs(List<String> files,
+      List<JsonFileSpec> jsonFiles) throws FlagUsageException, IOException {
+    return createInputs(files, new ArrayList<String>() /* zips */, jsonFiles, false);
+  }
+
   /**
    * Creates inputs from a list of source files and zips.
    *
@@ -585,6 +624,25 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
    */
   protected List<SourceFile> createInputs(List<String> files,
       List<String> zips, boolean allowStdIn) throws FlagUsageException, IOException {
+    return createInputs(files, zips, null, allowStdIn);
+  }
+
+  /**
+   * Creates inputs from a list of source files, zips and json files.
+   *
+   * Can be overridden by subclasses who want to pull files from different
+   * places.
+   *
+   * @param files A list of filenames.
+   * @param zips A list of zip filenames.
+   * @param jsonFiles A list of json encoded files.
+   * @param allowStdIn Whether '-' is allowed appear as a filename to represent
+   *        stdin. If true, '-' is only allowed to appear once.
+   * @return An array of inputs
+   */
+  protected List<SourceFile> createInputs(List<String> files,
+      List<String> zips, List<JsonFileSpec> jsonFiles, boolean allowStdIn)
+      throws FlagUsageException, IOException {
     List<SourceFile> inputs = new ArrayList<>(files.size());
     boolean usingStdin = false;
     for (String filename : files) {
@@ -619,23 +677,34 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
         inputs.addAll(newFiles);
       }
     }
+    if (jsonFiles != null) {
+      for (JsonFileSpec jsonFile : jsonFiles) {
+        inputs.add(SourceFile.fromCode(jsonFile.getPath(), jsonFile.getSrc()));
+      }
+    }
     return inputs;
   }
 
   /**
    * Creates JS source code inputs from a list of files.
    */
-  private List<SourceFile> createSourceInputs(List<String> files, List<String> zips)
+  private List<SourceFile> createSourceInputs(List<String> files, List<String> zips,
+      List<JsonFileSpec> jsonFiles)
       throws FlagUsageException, IOException {
     if (isInTestMode()) {
-      return inputsSupplierForTesting.get();
+      return inputsSupplierForTesting != null ? inputsSupplierForTesting.get()
+          : null;
     }
-    if (files.isEmpty() && zips.isEmpty()) {
+    if (files.isEmpty() && zips.isEmpty() && jsonFiles == null) {
       // Request to read from stdin.
       files = Collections.singletonList("-");
     }
     try {
-      return createInputs(files, zips, true);
+      if (jsonFiles != null) {
+        return createInputs(files, jsonFiles);
+      } else {
+        return createInputs(files, zips, true);
+      }
     } catch (FlagUsageException e) {
       throw new FlagUsageException("Bad --js flag. " + e.getMessage());
     }
@@ -666,7 +735,7 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
    * @return An array of module objects
    */
   List<JSModule> createJsModules(
-      List<String> specs, List<String> jsFiles)
+      List<String> specs, List<SourceFile> inputs)
       throws FlagUsageException, IOException {
     if (isInTestMode()) {
       return modulesSupplierForTesting.get();
@@ -674,7 +743,7 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
 
     Preconditions.checkState(specs != null);
     Preconditions.checkState(!specs.isEmpty());
-    Preconditions.checkState(jsFiles != null);
+    Preconditions.checkState(inputs != null);
 
     List<String> moduleNames = new ArrayList<>(specs.size());
     Map<String, JSModule> modulesByName = new LinkedHashMap<>();
@@ -755,7 +824,7 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
       modulesByName.put(name, module);
     }
 
-    final int totalNumJsFiles = jsFiles.size();
+    final int totalNumJsFiles = inputs.size();
 
     if (numJsFilesExpected >= 0 || minJsFilesRequired > totalNumJsFiles) {
       if (minJsFilesRequired > totalNumJsFiles) {
@@ -782,10 +851,9 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
         numJsFiles = numJsFilesLeft;
       }
 
-      List<String> moduleJsFiles =
-          jsFiles.subList(numJsFilesLeft - numJsFiles, numJsFilesLeft);
-      for (SourceFile input :
-          createInputs(moduleJsFiles, false)) {
+      List<SourceFile> moduleFiles =
+          inputs.subList(numJsFilesLeft - numJsFiles, numJsFilesLeft);
+      for (SourceFile input : moduleFiles) {
         module.add(input);
       }
       numJsFilesLeft -= numJsFiles;
@@ -916,11 +984,11 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
    * path prefix.
    */
   private static void maybeCreateDirsForPath(String pathPrefix) {
-    if (!Strings.isNullOrEmpty(pathPrefix)) {
+    if (pathPrefix.length() > 0) {
       String dirName =
           pathPrefix.charAt(pathPrefix.length() - 1) == File.separatorChar
-              ? pathPrefix.substring(0, pathPrefix.length() - 1)
-              : new File(pathPrefix).getParent();
+              ? pathPrefix.substring(0, pathPrefix.length() - 1) : new File(
+                  pathPrefix).getParent();
       if (dirName != null) {
         new File(dirName).mkdirs();
       }
@@ -971,6 +1039,30 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
 
     List<String> jsFiles = config.js;
     List<String> moduleSpecs = config.module;
+    List<JsonFileSpec> jsonFiles = null;
+
+    if (config.useJsonStreams) {
+      jsonFiles = parseJsonFilesFromInputStream();
+
+      ImmutableMap.Builder<String, SourceMapInput> inputSourceMaps
+          = new ImmutableMap.Builder<>();
+
+      boolean foundJsonInputSourceMap = false;
+      for (JsonFileSpec jsonFile : jsonFiles) {
+        if (jsonFile.getSourceMap() != null && jsonFile.getSourceMap().length() > 0) {
+          String sourceMapPath = jsonFile.getPath() + ".map";
+          SourceFile sourceMap = SourceFile.fromCode(sourceMapPath,
+              jsonFile.getSourceMap());
+          inputSourceMaps.put(sourceMapPath, new SourceMapInput(sourceMap));
+          foundJsonInputSourceMap = true;
+        }
+      }
+
+      if (foundJsonInputSourceMap) {
+        inputSourceMaps.putAll(options.inputSourceMaps);
+        options.inputSourceMaps = inputSourceMaps.build();
+      }
+    }
 
     boolean createCommonJsModules = false;
     if (options.processCommonJSModules
@@ -978,8 +1070,9 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
       createCommonJsModules = true;
       moduleSpecs.remove(0);
     }
+    List<SourceFile> inputs = createSourceInputs(jsFiles, config.jsZip, jsonFiles);
     if (!moduleSpecs.isEmpty()) {
-      modules = createJsModules(moduleSpecs, jsFiles);
+      modules = createJsModules(moduleSpecs, inputs);
       for (JSModule m : modules) {
         outputFileNames.add(getModuleOutputFileName(m));
       }
@@ -990,7 +1083,6 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
         result = compiler.compileModules(externs, modules, options);
       }
     } else {
-      List<SourceFile> inputs = createSourceInputs(jsFiles, config.jsZip);
       if (config.skipNormalOutputs) {
         compiler.init(externs, inputs, options);
         compiler.hoistExterns();
@@ -1074,10 +1166,14 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
     } else if (!options.checksOnly && result.success) {
       outputModuleGraphJson();
       if (modules == null) {
-        outputSingleBinary();
+        outputSingleBinary(options);
 
         // Output the source map if requested.
-        outputSourceMap(options, config.jsOutputFile);
+        // If output files are being written to stdout as a JSON string,
+        // outputSingleBinary will have added the sourcemap to the output file
+        if (!config.useJsonStreams) {
+          outputSourceMap(options, config.jsOutputFile);
+        }
       } else {
         outputModuleBinaryAndSourceMaps(modules, options);
       }
@@ -1095,6 +1191,10 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
       // Output the manifest and bundle files if requested.
       outputManifest();
       outputBundle();
+
+      if (config.useJsonStreams) {
+        outputJsonStream();
+      }
     }
 
     // return 0 if no errors, the error count otherwise
@@ -1105,7 +1205,7 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
     return SourceCodeEscapers.javascriptEscaper().asFunction();
   }
 
-  void outputSingleBinary() throws IOException {
+  void outputSingleBinary(B options) throws IOException {
     Function<String, String> escaper = null;
     String marker = OUTPUT_MARKER;
     if (config.outputWrapper.contains(OUTPUT_MARKER_JS_STRING)) {
@@ -1113,11 +1213,56 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
       escaper = getJavascriptEscaper();
     }
 
-    Appendable jsOutput = createDefaultOutput();
+    if (config.useJsonStreams) {
+      this.filesToStreamOut.add(createJsonFile(options, marker, escaper));
+    } else {
+      Appendable jsOutput = createDefaultOutput();
+      writeOutput(
+          jsOutput, compiler, compiler.toSource(), config.outputWrapper,
+          marker, escaper);
+      closeAppendable(jsOutput);
+    }
+  }
+
+  /**
+   * Save the compiler output to a JsonFileSpec to be later written to
+   * stdout
+   */
+  JsonFileSpec createJsonFile(B options, String outputMarker,
+      Function<String, String> escaper) throws IOException {
+    Appendable jsOutput = new StringBuilder();
     writeOutput(
         jsOutput, compiler, compiler.toSource(), config.outputWrapper,
-        marker, escaper);
-    closeAppendable(jsOutput);
+        outputMarker, escaper);
+
+    JsonFileSpec jsonOutput = new JsonFileSpec(jsOutput.toString(),
+        Strings.isNullOrEmpty(config.jsOutputFile) ?
+            "compiled.js" : config.jsOutputFile);
+
+    if (!Strings.isNullOrEmpty(options.sourceMapOutputPath)) {
+      StringBuilder sourcemap = new StringBuilder();
+      compiler.getSourceMap().appendTo(sourcemap, jsonOutput.getPath());
+      jsonOutput.setSourceMap(sourcemap.toString());
+    }
+
+    return jsonOutput;
+  }
+
+  void outputJsonStream() throws IOException {
+    JsonWriter jsonWriter = new JsonWriter(
+        new BufferedWriter(new OutputStreamWriter(defaultJsOutput, "UTF-8")));
+    jsonWriter.beginArray();
+    for (JsonFileSpec jsonFile : this.filesToStreamOut) {
+      jsonWriter.beginObject();
+      jsonWriter.name("src").value(jsonFile.getSrc());
+      jsonWriter.name("path").value(jsonFile.getPath());
+      if (!Strings.isNullOrEmpty(jsonFile.getSourceMap())) {
+        jsonWriter.name("source_map").value(jsonFile.getSourceMap());
+      }
+      jsonWriter.endObject();
+    }
+    jsonWriter.endArray();
+    jsonWriter.close();
   }
 
   private void outputModuleBinaryAndSourceMaps(
@@ -1130,38 +1275,70 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
     // If the source map path is in fact a pattern for each
     // module, create a stream per-module. Otherwise, create
     // a single source map.
-    Writer mapOut = null;
+    Writer mapFileOut = null;
 
-    if (!shouldGenerateMapPerModule(options)) {
-      mapOut = fileNameToOutputWriter2(expandSourceMapPath(options, null));
+    // When the json_streams flag is specified, sourcemaps are always generated
+    // per module
+    if (!(shouldGenerateMapPerModule(options) || config.useJsonStreams)) {
+      mapFileOut = fileNameToOutputWriter2(
+          expandSourceMapPath(options, null));
     }
 
     for (JSModule m : modules) {
-      if (shouldGenerateMapPerModule(options)) {
-        mapOut = fileNameToOutputWriter2(expandSourceMapPath(options, m));
-      }
-
-      try (Writer writer = fileNameToLegacyOutputWriter(getModuleOutputFileName(m))) {
-        if (options.sourceMapOutputPath != null) {
-          compiler.getSourceMap().reset();
+      if (config.useJsonStreams) {
+        this.filesToStreamOut.add(createJsonFileFromModule(m));
+      } else {
+        if (shouldGenerateMapPerModule(options)) {
+          mapFileOut = fileNameToOutputWriter2(expandSourceMapPath(options, m));
         }
-        writeModuleOutput(writer, m);
-        if (options.sourceMapOutputPath != null) {
-          compiler.getSourceMap().appendTo(mapOut, m.getName());
+
+        try (Writer writer = fileNameToLegacyOutputWriter(
+            getModuleOutputFileName(m))) {
+          if (options.sourceMapOutputPath != null) {
+            compiler.getSourceMap().reset();
+          }
+          writeModuleOutput(writer, m);
+          if (options.sourceMapOutputPath != null) {
+            compiler.getSourceMap().appendTo(mapFileOut, m.getName());
+          }
         }
-      }
 
-      if (shouldGenerateMapPerModule(options) && mapOut != null) {
-        mapOut.close();
-        mapOut = null;
+        if (shouldGenerateMapPerModule(options) && mapFileOut != null) {
+          mapFileOut.close();
+          mapFileOut = null;
+        }
       }
     }
 
-    if (mapOut != null) {
-      mapOut.close();
+    if (mapFileOut != null) {
+      mapFileOut.close();
     }
   }
 
+  /**
+   * Given an output module, convert it to a JSONFileSpec with associated
+   * sourcemap
+   */
+  private JsonFileSpec createJsonFileFromModule(JSModule module) throws
+      FlagUsageException, IOException{
+    compiler.getSourceMap().reset();
+
+    StringBuilder output = new StringBuilder();
+    writeModuleOutput(output, module);
+
+    JsonFileSpec jsonFile = new JsonFileSpec(output.toString(),
+        getModuleOutputFileName(module));
+
+    StringBuilder moduleSourceMap = new StringBuilder();
+
+    compiler.getSourceMap().appendTo(moduleSourceMap,
+        getModuleOutputFileName(module));
+
+    jsonFile.setSourceMap(moduleSourceMap.toString());
+
+    return jsonFile;
+  }
+
   /**
    * Query the flag for the input charset, and return a Charset object
    * representing the selection.
@@ -2266,6 +2443,13 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
         return this;
     }
 
+    private boolean useJsonStreams = false;
+
+    CommandLineConfig setUseJsonStreams(boolean useJsonStreams) {
+      this.useJsonStreams = useJsonStreams;
+      return this;
+    }
+
   }
 
   /**
@@ -2294,4 +2478,37 @@ public abstract class AbstractCommandLineRunner<A extends Compiler,
       entries.clear();
     }
   }
+
+  /**
+   * Representation of a source file from an encoded json stream input
+   */
+  private class JsonFileSpec {
+    private final String src;
+    private final String path;
+    private String source_map;
+
+    public JsonFileSpec(String src, String path) {
+      this(src, path, null);
+    }
+
+    public JsonFileSpec(String src, String path, String source_map) {
+      this.src = src;
+      this.path = path;
+      this.source_map = source_map;
+    }
+
+    public String getSrc() {
+      return this.src;
+    }
+
+    public String getPath() {
+      return this.path;
+    }
+
+    public String getSourceMap() { return this.source_map; }
+
+    public void setSourceMap(String map) {
+      this.source_map = map;
+    }
+  }
 }
diff --git a/src/com/google/javascript/jscomp/CheckJSDoc.java b/src/com/google/javascript/jscomp/CheckJSDoc.java
index b6f6c3705..f346f9b9d 100644
--- a/src/com/google/javascript/jscomp/CheckJSDoc.java
+++ b/src/com/google/javascript/jscomp/CheckJSDoc.java
@@ -27,7 +27,7 @@ import com.google.javascript.rhino.Token;
  *
  * @author chadkillingsworth@gmail.com (Chad Killingsworth)
  */
-final class CheckJSDoc extends AbstractPostOrderCallback implements HotSwapCompilerPass {
+final class CheckJSDoc extends AbstractPostOrderCallback implements CompilerPass {
 
   public static final DiagnosticType MISPLACED_MSG_ANNOTATION =
       DiagnosticType.disabled("JSC_MISPLACED_MSG_ANNOTATION",
@@ -66,11 +66,6 @@ final class CheckJSDoc extends AbstractPostOrderCallback implements HotSwapCompi
     NodeTraversal.traverseEs6(compiler, root, this);
   }
 
-  @Override
-  public void hotSwapScript(Node scriptRoot, Node originalRoot) {
-    NodeTraversal.traverseEs6(compiler, scriptRoot, this);
-  }
-
   @Override
   public void visit(NodeTraversal t, Node n, Node parent) {
     JSDocInfo info = n.getJSDocInfo();
diff --git a/src/com/google/javascript/jscomp/CommandLineRunner.java b/src/com/google/javascript/jscomp/CommandLineRunner.java
index 7cb0d75d3..01a0506b5 100644
--- a/src/com/google/javascript/jscomp/CommandLineRunner.java
+++ b/src/com/google/javascript/jscomp/CommandLineRunner.java
@@ -606,6 +606,14 @@ public class CommandLineRunner extends
             usage = "A file containing an instrumentation template.")
         private String instrumentationFile = "";
 
+    @Option(name = "--json_streams",
+        hidden = true,
+        handler = BooleanOptionHandler.class,
+        usage = "Specifies that standard input and output streams will be "
+            + "a JSON array of sources. Each source will be an object of the "
+            + "form {path: filename, src: file_contents, srcmap: srcmap_contents }. "
+            + "Intended for use by stream-based build systems such as gulpjs.")
+    private boolean jsonStreams = false;
 
     @Argument
     private List<String> arguments = new ArrayList<>();
@@ -1123,7 +1131,8 @@ public class CommandLineRunner extends
           .setAngularPass(flags.angularPass)
           .setTracerMode(flags.tracerMode)
           .setInstrumentationTemplateFile(flags.instrumentationFile)
-          .setNewTypeInference(flags.useNewTypeInference);
+          .setNewTypeInference(flags.useNewTypeInference)
+          .setUseJsonStreams(flags.jsonStreams);
     }
     errorStream = null;
   }
diff --git a/src/com/google/javascript/jscomp/DefaultPassConfig.java b/src/com/google/javascript/jscomp/DefaultPassConfig.java
index 29e9856c7..4f26ee15e 100644
--- a/src/com/google/javascript/jscomp/DefaultPassConfig.java
+++ b/src/com/google/javascript/jscomp/DefaultPassConfig.java
@@ -990,9 +990,9 @@ public final class DefaultPassConfig extends PassConfig {
           "functions are set.");
 
   /** Verifies JSDoc annotations are used properly. */
-  private final HotSwapPassFactory checkJsDoc = new HotSwapPassFactory("checkJsDoc", true) {
+  private final PassFactory checkJsDoc = new PassFactory("checkJsDoc", true) {
     @Override
-    protected HotSwapCompilerPass create(AbstractCompiler compiler) {
+    protected CompilerPass create(AbstractCompiler compiler) {
       return new CheckJSDoc(compiler);
     }
   };
@@ -1158,10 +1158,10 @@ public final class DefaultPassConfig extends PassConfig {
     }
   };
 
-  private final HotSwapPassFactory es6RewriteDestructuring =
-      new HotSwapPassFactory("Es6RewriteDestructuring", true) {
+  private final PassFactory es6RewriteDestructuring =
+      new PassFactory("Es6RewriteDestructuring", true) {
         @Override
-        protected HotSwapCompilerPass create(final AbstractCompiler compiler) {
+        protected CompilerPass create(final AbstractCompiler compiler) {
           return new Es6RewriteDestructuring(compiler);
         }
       };
@@ -1174,10 +1174,10 @@ public final class DefaultPassConfig extends PassConfig {
         }
       };
 
-  private final HotSwapPassFactory es6RewriteArrowFunction =
-      new HotSwapPassFactory("Es6RewriteArrowFunction", true) {
+  private final PassFactory es6RewriteArrowFunction =
+      new PassFactory("Es6RewriteArrowFunction", true) {
         @Override
-        protected HotSwapCompilerPass create(final AbstractCompiler compiler) {
+        protected CompilerPass create(final AbstractCompiler compiler) {
           return new Es6RewriteArrowFunction(compiler);
         }
       };
diff --git a/src/com/google/javascript/jscomp/GlobalTypeInfo.java b/src/com/google/javascript/jscomp/GlobalTypeInfo.java
index 78ab9d70b..302eea49a 100644
--- a/src/com/google/javascript/jscomp/GlobalTypeInfo.java
+++ b/src/com/google/javascript/jscomp/GlobalTypeInfo.java
@@ -249,7 +249,10 @@ class GlobalTypeInfo implements CompilerPass {
       UNDECLARED_NAMESPACE,
       UNKNOWN_OVERRIDE,
       UNRECOGNIZED_TYPE_NAME,
-      WRONG_PARAMETER_COUNT
+      WRONG_PARAMETER_COUNT//,
+      // VarCheck.UNDEFINED_VAR_ERROR,
+      // VariableReferenceCheck.REDECLARED_VARIABLE,
+      // VariableReferenceCheck.EARLY_REFERENCE
       );
 
   // An out-to-in list of the scopes, built during CollectNamedTypes
@@ -632,7 +635,6 @@ class GlobalTypeInfo implements CompilerPass {
         }
         case Token.VAR: {
           Node nameNode = n.getFirstChild();
-          String varName = nameNode.getString();
           if (NodeUtil.isNamespaceDecl(nameNode)) {
             visitObjlitNamespace(nameNode);
           } else if (NodeUtil.isTypedefDecl(nameNode)) {
@@ -641,17 +643,11 @@ class GlobalTypeInfo implements CompilerPass {
             visitEnum(nameNode);
           } else if (isAliasedNamespaceDefinition(nameNode)) {
             visitAliasedNamespace(nameNode);
-          } else if (varName.equals(WINDOW)
+          } else if (nameNode.getString().equals(WINDOW)
               && nameNode.isFromExterns()
               && !this.currentScope.isDefinedLocally(WINDOW, false)) {
             this.currentScope.addLocal(
-                WINDOW,
-                getVarTypeFromAnnotation(nameNode, this.currentScope),
-                false, true);
-          } else if (this.currentScope.isFunction()
-              && !this.currentScope.isDefinedLocally(varName, false)) {
-            // Add a dummy local to avoid shadowing errors
-            this.currentScope.addLocal(varName, JSType.UNKNOWN, false, false);
+                WINDOW, getVarTypeFromAnnotation(nameNode, this.currentScope), false, true);
           }
           break;
         }
@@ -778,6 +774,13 @@ class GlobalTypeInfo implements CompilerPass {
       if (NodeUtil.getRValueOfLValue(qnameNode) != null) {
         warnings.add(JSError.make(qnameNode, CANNOT_INIT_TYPEDEF));
       }
+      // if (qnameNode.isName()
+      //     && currentScope.isDefinedLocally(qnameNode.getString())) {
+      //   warnings.add(JSError.make(
+      //       qnameNode,
+      //       VariableReferenceCheck.REDECLARED_VARIABLE,
+      //       qnameNode.getQualifiedName()));
+      // }
       if (currentScope.isDefined(qnameNode)) {
         return;
       }
@@ -789,6 +792,12 @@ class GlobalTypeInfo implements CompilerPass {
     private void visitEnum(Node qnameNode) {
       Preconditions.checkState(qnameNode.isQualifiedName());
       qnameNode.putBooleanProp(Node.ANALYZED_DURING_GTI, true);
+      // if (qnameNode.isName()
+      //     && currentScope.isDefinedLocally(qnameNode.getString())) {
+      //   String qname = qnameNode.getQualifiedName();
+      //   warnings.add(JSError.make(qnameNode,
+      //           VariableReferenceCheck.REDECLARED_VARIABLE, qname));
+      // }
       if (currentScope.isDefined(qnameNode)) {
         return;
       }
@@ -858,6 +867,9 @@ class GlobalTypeInfo implements CompilerPass {
       } else if (currentScope.isDefinedLocally(nameNode.getString(), false)) {
         String fnName = nameNode.getString();
         Preconditions.checkState(!fnName.contains("."));
+        // warnings.add(JSError.make(
+        //     fn, VariableReferenceCheck.REDECLARED_VARIABLE, fnName));
+        // Redeclared variables also need gensymed names
         internalName = ANON_FUN_PREFIX + freshId;
         anonFunNames.put(fn, internalName);
         freshId++;
@@ -1014,10 +1026,17 @@ class GlobalTypeInfo implements CompilerPass {
 
   private class ProcessScope extends AbstractShallowCallback {
     private final NTIScope currentScope;
+    // /**
+    //  * Keep track of undeclared vars as they are crawled to warn about
+    //  * use before declaration and undeclared variables.
+    //  * We use a multimap so we can give all warnings rather than just the first.
+    //  */
+    // private final Multimap<String, Node> undeclaredVars;
     private Set<Node> lendsObjlits = new LinkedHashSet<>();
 
     ProcessScope(NTIScope currentScope) {
       this.currentScope = currentScope;
+      // this.undeclaredVars = LinkedHashMultimap.create();
     }
 
     void finishProcessingScope() {
@@ -1025,6 +1044,11 @@ class GlobalTypeInfo implements CompilerPass {
         processLendsNode(objlit);
       }
       lendsObjlits = null;
+
+      // for (Node nameNode : undeclaredVars.values()) {
+      //   warnings.add(JSError.make(nameNode,
+      //         VarCheck.UNDEFINED_VAR_ERROR, nameNode.getString()));
+      // }
     }
 
     /**
@@ -1118,6 +1142,7 @@ class GlobalTypeInfo implements CompilerPass {
               currentScope.getTypedef(name) != null
               || !name.equals(currentScope.getName())
               && !currentScope.isDefinedLocally(name, false)) {
+            // undeclaredVars.put(name, n);
           }
           break;
         }
@@ -1148,12 +1173,10 @@ class GlobalTypeInfo implements CompilerPass {
 
     private void visitVar(Node nameNode, Node parent) {
       String name = nameNode.getString();
-      boolean isDefinedLocally = this.currentScope.isDefinedLocally(name, false);
-      if (isDefinedLocally && this.currentScope.isNamespace(name)) {
-        return;
-      }
-      if (NodeUtil.isTypedefDecl(nameNode) || NodeUtil.isEnumDecl(nameNode)) {
-        if (!isDefinedLocally) {
+      if (this.currentScope.isNamespace(name)
+          || NodeUtil.isTypedefDecl(nameNode)
+          || NodeUtil.isEnumDecl(nameNode)) {
+        if (!this.currentScope.isDefinedLocally(name, false)) {
           // Malformed enum or typedef
           this.currentScope.addLocal(
               name, JSType.UNKNOWN, false, nameNode.isFromExterns());
@@ -1163,7 +1186,16 @@ class GlobalTypeInfo implements CompilerPass {
       Node initializer = nameNode.getFirstChild();
       if (initializer != null && initializer.isFunction()) {
         return;
+      } else if (this.currentScope.isDefinedLocally(name, false)) {
+        // warnings.add(JSError.make(
+        //     n, VariableReferenceCheck.REDECLARED_VARIABLE, name));
+        return;
       }
+      // for (Node useBeforeDeclNode : undeclaredVars.get(name)) {
+      //   warnings.add(JSError.make(useBeforeDeclNode,
+      //       VariableReferenceCheck.EARLY_REFERENCE, name));
+      // }
+      // undeclaredVars.removeAll(name);
       if (parent.isCatch()) {
         this.currentScope.addLocal(name, JSType.UNKNOWN, false, false);
       } else {
@@ -1265,6 +1297,10 @@ class GlobalTypeInfo implements CompilerPass {
     /** Compute the declared type for a given scope. */
     private NTIScope visitFunctionLate(Node fn, RawNominalType ownerType) {
       Preconditions.checkArgument(fn.isFunction());
+      // String fnName = NodeUtil.getFunctionName(fn);
+      // if (fnName != null && !fnName.contains(".")) {
+      //   undeclaredVars.removeAll(fnName);
+      // }
       String internalName = getFunInternalName(fn);
       NTIScope fnScope = currentScope.getScope(internalName);
       JSDocInfo jsdoc = NodeUtil.getBestJSDocInfo(fn);
@@ -1781,8 +1817,7 @@ class GlobalTypeInfo implements CompilerPass {
       Node qnameNode = declNode.isGetProp()
           ? declNode : NodeUtil.getFunctionNameNode(declNode);
       if (result.slotType != null && qnameNode != null && qnameNode.isName()) {
-        parentScope.addLocal(qnameNode.getString(),
-            result.slotType, false, qnameNode.isFromExterns());
+        parentScope.addSimpleType(qnameNode, result.slotType);
       }
       if (ctorType != null) {
         ctorType.setCtorFunction(result.functionType.toFunctionType(), commonTypes);
diff --git a/src/com/google/javascript/jscomp/NTIScope.java b/src/com/google/javascript/jscomp/NTIScope.java
index d4843fe57..02293da73 100644
--- a/src/com/google/javascript/jscomp/NTIScope.java
+++ b/src/com/google/javascript/jscomp/NTIScope.java
@@ -405,14 +405,21 @@ final class NTIScope implements DeclaredTypeRegistry {
     return ImmutableSet.copyOf(externs.keySet());
   }
 
-  // We don't check for duplicates here, mainly because we add some
-  // intentionally during the two phases of GlobalTypeInfo.
-  // If a variable is declared many times in a scope, the last definition
-  // overwrites the previous ones. For correctness, we rely on the fact that
-  // the var-check passes run before type checking.
+  // Like addLocal, but used when the type is already defined locally
+  void addSimpleType(Node qnameNode, JSType declType) {
+    Preconditions.checkState(qnameNode.isName());
+    String name = qnameNode.getString();
+    if (qnameNode.isFromExterns()) {
+      externs.put(name, declType);
+    } else {
+      locals.put(name, declType);
+    }
+  }
+
   void addLocal(String name, JSType declType,
       boolean isConstant, boolean isFromExterns) {
     Preconditions.checkArgument(!name.contains("."));
+    Preconditions.checkArgument(!isDefinedLocally(name, false));
     if (isConstant) {
       constVars.add(name);
     }
@@ -428,10 +435,9 @@ final class NTIScope implements DeclaredTypeRegistry {
   }
 
   private void addNamespaceLit(Node qnameNode, NamespaceLit nslit) {
+    Preconditions.checkArgument(!isNamespace(qnameNode));
     if (qnameNode.isName()) {
       String varName = qnameNode.getString();
-      Preconditions.checkArgument(
-          !isDefinedLocally(varName, false) || !isNamespace(qnameNode));
       localNamespaces.put(varName, nslit);
       if (qnameNode.isFromExterns() && !externs.containsKey(varName)) {
         // We don't know the full type of a namespace until after we see all
@@ -440,7 +446,6 @@ final class NTIScope implements DeclaredTypeRegistry {
         externs.put(qnameNode.getString(), null);
       }
     } else {
-      Preconditions.checkArgument(!isNamespace(qnameNode));
       QualifiedName qname = QualifiedName.fromNode(qnameNode);
       Namespace ns = getNamespace(qname.getLeftmostName());
       ns.addSubnamespace(qname.getAllButLeftmost(), nslit);
diff --git a/src/com/google/javascript/jscomp/ProcessEs6Modules.java b/src/com/google/javascript/jscomp/ProcessEs6Modules.java
index b3278af05..5a2ac57b7 100644
--- a/src/com/google/javascript/jscomp/ProcessEs6Modules.java
+++ b/src/com/google/javascript/jscomp/ProcessEs6Modules.java
@@ -53,7 +53,7 @@ public final class ProcessEs6Modules extends AbstractPostOrderCallback {
   static final DiagnosticType USELESS_USE_STRICT_DIRECTIVE =
       DiagnosticType.warning(
           "JSC_USELESS_USE_STRICT_DIRECTIVE",
-          "'use strict' is unnecessary in ES6 modules.");
+          "'use strict' is unnecessary in goog.module files.");
 
   static final DiagnosticType NAMESPACE_IMPORT_CANNOT_USE_STAR =
       DiagnosticType.error(


Test patch:
diff --git a/test/com/google/javascript/jscomp/NewTypeInferenceES5OrLowerTest.java b/test/com/google/javascript/jscomp/NewTypeInferenceES5OrLowerTest.java
index 1bd40b61f..4c4434ef6 100644
--- a/test/com/google/javascript/jscomp/NewTypeInferenceES5OrLowerTest.java
+++ b/test/com/google/javascript/jscomp/NewTypeInferenceES5OrLowerTest.java
@@ -47,8 +47,7 @@ public final class NewTypeInferenceES5OrLowerTest extends NewTypeInferenceTestBa
 
     typeCheckCustomExterns(
         DEFAULT_EXTERNS + "var maybeStr; /** @type {string} */ var maybeStr;",
-        "maybeStr - 5;",
-        NewTypeInference.INVALID_OPERAND_TYPE);
+        "maybeStr - 5;");
 
     typeCheckCustomExterns(
         DEFAULT_EXTERNS + "/** @type {string} */ var str;", "str - 5;",
@@ -14349,44 +14348,4 @@ public final class NewTypeInferenceES5OrLowerTest extends NewTypeInferenceTestBa
         "MyArray.prototype.m = function(x) {};",
         "(new MyArray(123)).m(new MyArray('asdf'));"));
   }
-
-  public void testDontCrashWhenShadowingANamespace() {
-    typeCheck(Joiner.on('\n').join(
-        "/** @const */",
-        "var ns = {};",
-        "function f() {",
-        "  /** @const */",
-        "  var ns = ns || {};",
-        "  /** @constructor */",
-        "  ns.Foo = function() {};",
-        "}"));
-
-    typeCheck(Joiner.on('\n').join(
-        "/** @const */",
-        "var ns = {};",
-        "function f() {",
-        "  function ns() {};",
-        "  /** @constructor */",
-        "  ns.Foo = function() {};",
-        "}"));
-
-    typeCheck(Joiner.on('\n').join(
-        "/** @const */",
-        "var ns = {};",
-        "function f() {",
-        "  var ns = {};",
-        "  /** @constructor */",
-        "  ns.Foo = function() {};",
-        "}"));
-
-    typeCheck(Joiner.on('\n').join(
-        "function ns() {};",
-        "/** @type {number} */",
-        "ns.prop = 123;",
-        "function f() {",
-        "  var ns = {};",
-        "  /** @constructor */",
-        "  ns.Foo = function() {};",
-        "}"));
-  }
 }