Repository: kythe/kythe
Pull request #5073 (2021-09-14 15:33:18)

Modify the Kythe Java Indexer to add support for inline encoded GeneratedCodeInfo protos specified in a generated Java file.
These changes allow generated Java files with inline metadata to have Kythe edges to their source files.

shahms: /gcbrun
shahms: /gcbrun
shahms: /gcbrun
shahms: Would you mind running google-java-format on the changed files to satisfy the lint checks?
disha-google: I ran google-java-format on kythe/java/com/google/devtools/kythe/analyzers/java/KytheTreeScanner.java and kythe/java/com/google/devtools/kythe/platform/shared/KytheInlineMetadataLoader.java, but running it on kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java will change the byte values and break the tests. I can't see the lint error message, but is that a problem?
shahms: /gcbrun
shahms: > I ran google-java-format on kythe/java/com/google/devtools/kythe/analyzers/java/KytheTreeScanner.java and kythe/java/com/google/devtools/kythe/platform/shared/KytheInlineMetadataLoader.java, but running it on kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java will change the byte values and break the tests. I can't see the lint error message, but is that a problem?

I believe we exclude the testdata directories from the lint requirement for this reason.

Patch:
diff --git a/kythe/java/com/google/devtools/kythe/analyzers/java/JavaIndexer.java b/kythe/java/com/google/devtools/kythe/analyzers/java/JavaIndexer.java
index 46fe70bb4..bab622ef8 100644
--- a/kythe/java/com/google/devtools/kythe/analyzers/java/JavaIndexer.java
+++ b/kythe/java/com/google/devtools/kythe/analyzers/java/JavaIndexer.java
@@ -30,6 +30,7 @@ import com.google.devtools.kythe.platform.kzip.KZipException;
 import com.google.devtools.kythe.platform.kzip.KZipReader;
 import com.google.devtools.kythe.platform.shared.AnalysisException;
 import com.google.devtools.kythe.platform.shared.FileDataCache;
+import com.google.devtools.kythe.platform.shared.KytheInlineMetadataLoader;
 import com.google.devtools.kythe.platform.shared.KytheMetadataLoader;
 import com.google.devtools.kythe.platform.shared.MemoryStatisticsCollector;
 import com.google.devtools.kythe.platform.shared.MetadataLoaders;
@@ -162,6 +163,7 @@ public class JavaIndexer {
     metadataLoaders.addLoader(
         new ProtobufMetadataLoader(desc.getCompilationUnit(), config.getDefaultMetadataCorpus()));
     metadataLoaders.addLoader(new KytheMetadataLoader());
+    metadataLoaders.addLoader(new KytheInlineMetadataLoader());
 
     KytheJavacAnalyzer analyzer =
         new KytheJavacAnalyzer(
diff --git a/kythe/java/com/google/devtools/kythe/analyzers/java/KytheTreeScanner.java b/kythe/java/com/google/devtools/kythe/analyzers/java/KytheTreeScanner.java
index e99320cb4..4465c050f 100644
--- a/kythe/java/com/google/devtools/kythe/analyzers/java/KytheTreeScanner.java
+++ b/kythe/java/com/google/devtools/kythe/analyzers/java/KytheTreeScanner.java
@@ -16,6 +16,8 @@
 
 package com.google.devtools.kythe.analyzers.java;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.google.common.base.Ascii;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -1596,6 +1598,40 @@ public class KytheTreeScanner extends JCTreeScanner<JavaNode, TreeContext> {
     }
   }
 
+  private void loadAnnotationsData(String fullPath, String metadataComment) {
+    Metadata newMetadata = metadataLoaders.parseFile(fullPath, metadataComment.getBytes(UTF_8));
+    if (newMetadata == null) {
+      logger.atWarning().log("Can't load metadata %s", fullPath);
+      return;
+    }
+    metadata.add(newMetadata);
+    metadataFilePaths.add(fullPath);
+  }
+
+  private void loadInlineMetadata(String metadataPrefix) {
+    metadataPrefix += ":";
+    String fullPath = filePositions.getSourceFile().toUri().getPath();
+    try {
+      if (metadataFilePaths.contains(fullPath)) {
+        return;
+      }
+      for (List<Comment> commentList : comments.values()) {
+        for (Comment comment : commentList) {
+          int index = comment.text.indexOf(metadataPrefix);
+          if (index != -1) {
+            loadAnnotationsData(
+                fullPath,
+                Metadata.ANNOTATION_COMMENT_INLINE_METADATA_PREFIX
+                    + comment.text.substring(index + metadataPrefix.length()));
+            break;
+          }
+        }
+      }
+    } catch (IllegalArgumentException ex) {
+      logger.atWarning().withCause(ex).log("Can't read metadata at %s", fullPath);
+    }
+  }
+
   private void loadAnnotationsFromClassDecl(JCClassDecl decl) {
     for (JCAnnotation annotation : decl.getModifiers().getAnnotations()) {
       Symbol annotationSymbol = null;
@@ -1624,6 +1660,9 @@ public class KytheTreeScanner extends JCTreeScanner<JavaNode, TreeContext> {
         String comments = (String) rhs.getValue();
         if (comments.startsWith(Metadata.ANNOTATION_COMMENT_PREFIX)) {
           loadAnnotationsFile(comments.substring(Metadata.ANNOTATION_COMMENT_PREFIX.length()));
+        } else if (comments.startsWith(Metadata.ANNOTATION_COMMENT_INLINE_METADATA_PREFIX)) {
+          loadInlineMetadata(
+              comments.substring(Metadata.ANNOTATION_COMMENT_INLINE_METADATA_PREFIX.length()));
         }
       }
     }
diff --git a/kythe/java/com/google/devtools/kythe/platform/shared/BUILD b/kythe/java/com/google/devtools/kythe/platform/shared/BUILD
index 452d85e7f..4567478a4 100644
--- a/kythe/java/com/google/devtools/kythe/platform/shared/BUILD
+++ b/kythe/java/com/google/devtools/kythe/platform/shared/BUILD
@@ -33,6 +33,7 @@ java_library(
 java_library(
     name = "metadata",
     srcs = [
+        "KytheInlineMetadataLoader.java",
         "KytheMetadataLoader.java",
         "Metadata.java",
         "MetadataLoader.java",
@@ -45,6 +46,7 @@ java_library(
         "//kythe/java/com/google/devtools/kythe/common:flogger",
         "//kythe/java/com/google/devtools/kythe/util:json",
         "//kythe/proto:analysis_java_proto",
+        "//kythe/proto:metadata_java_proto",
         "//kythe/proto:storage_java_proto",
         "//third_party/guava",
         "@com_google_protobuf//:protobuf_java",
diff --git a/kythe/java/com/google/devtools/kythe/platform/shared/KytheInlineMetadataLoader.java b/kythe/java/com/google/devtools/kythe/platform/shared/KytheInlineMetadataLoader.java
new file mode 100644
index 000000000..2e9d2ac99
--- /dev/null
+++ b/kythe/java/com/google/devtools/kythe/platform/shared/KytheInlineMetadataLoader.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2016 The Kythe Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.devtools.kythe.platform.shared;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.kythe.analyzers.base.EdgeKind;
+import com.google.devtools.kythe.proto.Metadata.GeneratedCodeInfo;
+import com.google.devtools.kythe.proto.Metadata.MappingRule;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Loader that loads inline metadata from Java files generated by codegen that specify inline
+ * metadata.
+ *
+ * <p>The metadata is added as a comment on the last line of the Java file, as a base64 encoded
+ * string, which maps the Java symbols to the source language that the Java file was generated from.
+ */
+public final class KytheInlineMetadataLoader implements MetadataLoader {
+  private static final String ANNOTATION_PREFIX_STRING = "kythe-inline-metadata:";
+  private static final byte[] ANNOTATION_PREFIX = ANNOTATION_PREFIX_STRING.getBytes(UTF_8);
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  public KytheInlineMetadataLoader() {}
+
+  @Override
+  public Metadata parseFile(String fileName, byte[] data) {
+    if (!isCodegenFile(data)) {
+      return null;
+    }
+    GeneratedCodeInfo javaMetadata;
+    try {
+      javaMetadata = extractMetadata(data);
+    } catch (InvalidProtocolBufferException e) {
+      logger.atWarning().withCause(e).log(
+          "Error parsing GeneratedCodeInfo from file: %s", fileName);
+      return null;
+    }
+    if (javaMetadata == null) {
+      logger.atWarning().log("Error parsing GeneratedCodeInfo from file: %s", fileName);
+      return null;
+    }
+    return constructMetadata(javaMetadata);
+  }
+
+  private static boolean isCodegenFile(byte[] data) {
+    if (data == null || data.length < ANNOTATION_PREFIX.length) {
+      return false;
+    }
+    for (int i = 0; i < ANNOTATION_PREFIX.length; i++) {
+      if (data[i] != ANNOTATION_PREFIX[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private static GeneratedCodeInfo extractMetadata(byte[] data)
+      throws InvalidProtocolBufferException {
+    String metadata = new String(data, UTF_8).substring(ANNOTATION_PREFIX_STRING.length());
+    byte[] protoData = BaseEncoding.base64().decode(metadata);
+    return GeneratedCodeInfo.parseFrom(protoData);
+  }
+
+  private static Metadata constructMetadata(GeneratedCodeInfo javaMetadata) {
+    Metadata metadata = new Metadata();
+    for (MappingRule mapping : javaMetadata.getMetaList()) {
+      Metadata.Rule rule = new Metadata.Rule();
+      rule.begin = mapping.getBegin();
+      rule.end = mapping.getEnd();
+      rule.vname = mapping.getVname();
+      rule.edgeOut = EdgeKind.GENERATES;
+      rule.reverseEdge = true;
+      metadata.addRule(rule);
+    }
+    return metadata;
+  }
+}
diff --git a/kythe/java/com/google/devtools/kythe/platform/shared/Metadata.java b/kythe/java/com/google/devtools/kythe/platform/shared/Metadata.java
index a9a0b80d5..2f77bdf0d 100644
--- a/kythe/java/com/google/devtools/kythe/platform/shared/Metadata.java
+++ b/kythe/java/com/google/devtools/kythe/platform/shared/Metadata.java
@@ -93,4 +93,6 @@ public class Metadata {
    * participate in metadata rules.
    */
   public static final String ANNOTATION_COMMENT_PREFIX = "annotations:";
+
+  public static final String ANNOTATION_COMMENT_INLINE_METADATA_PREFIX = "kythe-inline-metadata:";
 }
diff --git a/kythe/proto/metadata.proto b/kythe/proto/metadata.proto
index 95d839cea..33ac8ad90 100644
--- a/kythe/proto/metadata.proto
+++ b/kythe/proto/metadata.proto
@@ -17,6 +17,8 @@ syntax = "proto3";
 
 package kythe.proto.metadata;
 
+option java_package = "com.google.devtools.kythe.proto";
+
 import "kythe/proto/storage.proto";
 
 // Schema for the JSON-encoded Kythe metadata describing the relationship
diff --git a/kythe/proto/metadata_go_proto/metadata.pb.go b/kythe/proto/metadata_go_proto/metadata.pb.go
index 4e3a29de6..d57dd0e6f 100644
--- a/kythe/proto/metadata_go_proto/metadata.pb.go
+++ b/kythe/proto/metadata_go_proto/metadata.pb.go
@@ -404,7 +404,9 @@ var file_kythe_proto_metadata_proto_rawDesc = []byte{
 	0x09, 0x53, 0x45, 0x4d, 0x41, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a,
 	0x53, 0x45, 0x4d, 0x41, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f,
 	0x53, 0x45, 0x4d, 0x41, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10,
-	0x02, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x02, 0x42, 0x21, 0x0a, 0x1f, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+	0x64, 0x65, 0x76, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x2e, 0x6b, 0x79, 0x74, 0x68, 0x65, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (


Test patch:
diff --git a/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/BUILD b/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/BUILD
index 2aaf28246..2a219f4a4 100644
--- a/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/BUILD
+++ b/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/BUILD
@@ -252,6 +252,18 @@ java_verifier_test(
     deps = ["@maven//:org_apache_tomcat_tomcat_annotations_api"],
 )
 
+java_verifier_test(
+    name = "inline_metadata_tests",
+    srcs = [
+        "InlineMetadata.java",
+    ],
+    indexer_opts = [
+        "--verbose",
+        "--default_metadata_corpus=default",
+    ],
+    deps = ["@maven//:org_apache_tomcat_tomcat_annotations_api"],
+)
+
 java_verifier_test(
     name = "processing_metadata_tests",
     srcs = [
diff --git a/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java b/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java
new file mode 100644
index 000000000..4846dfed9
--- /dev/null
+++ b/kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java
@@ -0,0 +1,76 @@
+
+
+
+// Autogenerated content.  DO NOT EDIT!
+
+package pkg;
+
+/**
+ * Experiment flags generated from:
+ * <ul>
+ *  <li>http://google3/googledata/experiments/fakefe/features/generic_feature.gcl
+ * </ul>
+ */
+
+@javax.annotation.Generated(
+  value = "//experiments/mendel/extensions/flag_codegen:codegen",
+  comments = "kythe-inline-metadata:Inline Codegen"
+)
+
+public final class InlineMetadata {
+  public static final class Generic {
+
+     /** All in all we're all just bricks in the wall. */
+    public static final String ANOTHER_BRICK_IN_THE_WALL = "Generic__another_brick_in_the_wall";
+
+     /** All in all we're all just bricks in the wall. */
+    public static final Boolean ANOTHER_BRICK_IN_THE_WALL_FLAG = false;
+
+     /** We sail tonight for Singapore. */
+    public static final String FALL_ASLEEP_WHILE_YOURE_ASHORE = "Generic__fall_asleep_while_youre_ashore";
+
+     /** We sail tonight for Singapore. */
+    public static final Boolean FALL_ASLEEP_WHILE_YOURE_ASHORE_FLAG = false;
+
+     /** Give me birds with broken wings. */
+    public static final String MEN_WITH_ALL_THE_ANSWERS = "Generic__men_with_all_the_answers";
+
+     /** Give me birds with broken wings. */
+    public static final Boolean MEN_WITH_ALL_THE_ANSWERS_FLAG = false;
+
+     /** A boolean that changes a button somewhere */
+    public static final String SOME_BUTTON = "Generic__some_button";
+
+     /** A boolean that changes a button somewhere */
+    public static final Boolean SOME_BUTTON_FLAG = false;
+
+     /** A boolean with default value=true. */
+    public static final String SOME_BUTTON_ROLLBACK = "Generic__some_button_rollback";
+
+     /** A boolean with default value=true. */
+    public static final Boolean SOME_BUTTON_ROLLBACK_FLAG = false;
+
+    private Generic() {}
+
+  }
+}
+
+//- EA.node/kind anchor
+//- EA.loc/start 416
+//- EA.loc/end 423
+//- EA defines/binding InlineMetadata
+//- vname("IDENTIFIER:Generic", google3, "", "googledata/experiments/fakefe/features/generic_feature.gcl", gcl) generates vname("9685fe12644670a48786e99ea2284102d5bebdb91672385b4efa2226d1777ad7", kythe, "", "kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java", "java")
+
+//- _Alt=vname("IDENTIFIER:Generic", google3, "", "googledata/experiments/fakefe/features/generic_feature.gcl", gcl) generates InlineMetadata
+//- InlineMetadata.node/kind record
+
+//- EB.node/kind anchor
+//- EB.loc/start 940
+//- EB.loc/end 975
+//- EB defines/binding FALL_ASLEEP_WHILE_YOURE_ASHORE_FLAG
+//- vname("IDENTIFIER:Generic.fall_asleep_while_youre_ashore", google3, "", "googledata/experiments/fakefe/features/generic_feature.gcl", gcl) generates vname("a0d784f4607cc6c9b38185145eb466fefcd7378ed6b54cb0041b1bec23bd20f3", kythe, "", "kythe/javatests/com/google/devtools/kythe/analyzers/java/testdata/pkg/InlineMetadata.java", "java")
+//- _Alt2=vname("IDENTIFIER:Generic.fall_asleep_while_youre_ashore", google3, "", "googledata/experiments/fakefe/features/generic_feature.gcl", gcl) generates FALL_ASLEEP_WHILE_YOURE_ASHORE_FLAG
+//- FALL_ASLEEP_WHILE_YOURE_ASHORE_FLAG.node/kind variable
+
+
+ // Inline Codegen:CAESfwgCEhUva3l0aGUvZWRnZS9nZW5lcmF0ZXMaXgoSSURFTlRJRklFUjpHZW5lcmljEgdnb29nbGUzIjpnb29nbGVkYXRhL2V4cGVyaW1lbnRzL2Zha2VmZS9mZWF0dXJlcy9nZW5lcmljX2ZlYXR1cmUuZ2NsKgNnY2wgoAMopwMSmQEIAhIVL2t5dGhlL2VkZ2UvZ2VuZXJhdGVzGngKLElERU5USUZJRVI6R2VuZXJpYy5hbm90aGVyX2JyaWNrX2luX3RoZV93YWxsEgdnb29nbGUzIjpnb29nbGVkYXRhL2V4cGVyaW1lbnRzL2Zha2VmZS9mZWF0dXJlcy9nZW5lcmljX2ZlYXR1cmUuZ2NsKgNnY2wgoQUovwUSngEIAhIVL2t5dGhlL2VkZ2UvZ2VuZXJhdGVzGn0KMUlERU5USUZJRVI6R2VuZXJpYy5mYWxsX2FzbGVlcF93aGlsZV95b3VyZV9hc2hvcmUSB2dvb2dsZTMiOmdvb2dsZWRhdGEvZXhwZXJpbWVudHMvZmFrZWZlL2ZlYXR1cmVzL2dlbmVyaWNfZmVhdHVyZS5nY2wqA2djbCCsByjPBxKYAQgCEhUva3l0aGUvZWRnZS9nZW5lcmF0ZXMadworSURFTlRJRklFUjpHZW5lcmljLm1lbl93aXRoX2FsbF90aGVfYW5zd2VycxIHZ29vZ2xlMyI6Z29vZ2xlZGF0YS9leHBlcmltZW50cy9mYWtlZmUvZmVhdHVyZXMvZ2VuZXJpY19mZWF0dXJlLmdjbCoDZ2NsILQJKNEJEosBCAISFS9reXRoZS9lZGdlL2dlbmVyYXRlcxpqCh5JREVOVElGSUVSOkdlbmVyaWMuc29tZV9idXR0b24SB2dvb2dsZTMiOmdvb2dsZWRhdGEvZXhwZXJpbWVudHMvZmFrZWZlL2ZlYXR1cmVzL2dlbmVyaWNfZmVhdHVyZS5nY2wqA2djbCCuCyi+CxKUAQgCEhUva3l0aGUvZWRnZS9nZW5lcmF0ZXMacwonSURFTlRJRklFUjpHZW5lcmljLnNvbWVfYnV0dG9uX3JvbGxiYWNrEgdnb29nbGUzIjpnb29nbGVkYXRhL2V4cGVyaW1lbnRzL2Zha2VmZS9mZWF0dXJlcy9nZW5lcmljX2ZlYXR1cmUuZ2NsKgNnY2wgnw0ouA0=