Repository: GoogleCloudPlatform/cloud-bigtable-client
Pull request #1838 (2018-06-04 21:59:01)

Use grpc-netty-shaded instead of grpc-netty.
@igorbernstein2, the `google-cloud-java` BOM uses `grpc-netty-shaded`, so this would be a prereq for using the BOM.  I'm running into trouble with the `Provider` which is expecting this:

* com.google.bigtable.repackaged.io.grpc.netty.shaded.com.google.bigtable.repackaged.io.grpc.netty.NettyChannelProvider

Do you have any thoughts on how to fix the service file?

sduskis: This should work towards fixing #1756 
igorbernstein2: Figured it out: the issue is caused by grpc-netty-shade.jar using the package name `io.grpc.netty.shaded.io.grpc.netty` and the  ServicesResourceTransformer usin greplaceAll:
https://github.com/apache/maven-shade-plugin/blob/ab82be37c5d4aa9855c03473a62cf7beed2f2b4a/src/main/java/org/apache/maven/plugins/shade/relocation/SimpleRelocator.java#L219

To workaround the issue, please the following, right before the relocation of `io.grpc`:

```
                <!-- Take special care of grpc-netty-shaded, it uses the package
                     io.grpc.netty.shaded.io.grpc.netty, which will cause the
                     ServicesResourceTransformer to replace both occurrences of io.grpc
                 -->
                <relocation>
                  <pattern>io.grpc.netty.shaded.io.grpc.netty</pattern>
                  <shadedPattern>com.google.bigtable.repackaged.io.grpc.netty</shadedPattern>
                </relocation>
```

Patch:
diff --git a/bigtable-client-core-parent/bigtable-client-core/pom.xml b/bigtable-client-core-parent/bigtable-client-core/pom.xml
index 01fe29fc2..96a74a5e2 100644
--- a/bigtable-client-core-parent/bigtable-client-core/pom.xml
+++ b/bigtable-client-core-parent/bigtable-client-core/pom.xml
@@ -167,11 +167,6 @@ limitations under the License.
             <version>${protobuff-java.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-tcnative-boringssl-static</artifactId>
-            <version>${netty-tcnative-boringssl-static.version}</version>
-        </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-protobuf</artifactId>
@@ -189,7 +184,7 @@ limitations under the License.
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
-            <artifactId>grpc-netty</artifactId>
+            <artifactId>grpc-netty-shaded</artifactId>
             <version>${grpc.version}</version>
         </dependency>
         <dependency>
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml
index f3e5dcb08..07ecf6c63 100644
--- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml
+++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml
@@ -175,6 +175,15 @@ limitations under the License.
                   <pattern>com.twitter</pattern>
                   <shadedPattern>com.google.bigtable.repackaged.com.twitter</shadedPattern>
                 </relocation>
+
+                <!-- Take special care of grpc-netty-shaded, it uses the package
+                     io.grpc.netty.shaded.io.grpc.netty, which will cause the
+                     ServicesResourceTransformer to replace both occurrences of io.grpc
+                 -->
+                <relocation>
+                  <pattern>io.grpc.netty.shaded.io.grpc.netty</pattern>
+                  <shadedPattern>com.google.bigtable.repackaged.io.grpc.netty</shadedPattern>
+                </relocation>
                 <relocation>
                   <pattern>io.grpc</pattern>
                   <shadedPattern>com.google.bigtable.repackaged.io.grpc</shadedPattern>
@@ -200,16 +209,12 @@ limitations under the License.
                     https://github.com/grpc/grpc-java/pull/2485
                 -->
                 <relocation>
-                  <pattern>io.netty</pattern>
-                  <shadedPattern>com.google.bigtable.repackaged.io.netty</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>META-INF/native/libnetty</pattern>
-                  <shadedPattern>META-INF/native/libcom_google_bigtable_repackaged_netty</shadedPattern>
+                  <pattern>META-INF/native/libio_grpc_netty_shaded_netty</pattern>
+                  <shadedPattern>META-INF/native/libcom_google_bigtable_repackaged_io_grpc_netty_shaded_netty</shadedPattern>
                 </relocation>
                 <relocation>
-                  <pattern>META-INF/native/netty</pattern>
-                  <shadedPattern>META-INF/native/com_google_bigtable_repackaged_netty</shadedPattern>
+                  <pattern>META-INF/native/io_grpc_netty_shaded_netty</pattern>
+                  <shadedPattern>META-INF/native/com_google_bigtable_repackaged_io_grpc_netty_shaded_netty</shadedPattern>
                 </relocation>
 
                 <relocation>
diff --git a/bigtable-hbase-parent/bigtable-hbase-1.x-shaded/pom.xml b/bigtable-hbase-parent/bigtable-hbase-1.x-shaded/pom.xml
index 048208cce..f991261b5 100644
--- a/bigtable-hbase-parent/bigtable-hbase-1.x-shaded/pom.xml
+++ b/bigtable-hbase-parent/bigtable-hbase-1.x-shaded/pom.xml
@@ -158,6 +158,14 @@ limitations under the License.
                   <pattern>com.twitter</pattern>
                   <shadedPattern>com.google.bigtable.repackaged.com.twitter</shadedPattern>
                 </relocation>
+                <!-- Take special care of grpc-netty-shaded, it uses the package
+                     io.grpc.netty.shaded.io.grpc.netty, which will cause the
+                     ServicesResourceTransformer to replace both occurrences of io.grpc
+                 -->
+                <relocation>
+                  <pattern>io.grpc.netty.shaded.io.grpc.netty</pattern>
+                  <shadedPattern>com.google.bigtable.repackaged.io.grpc.netty</shadedPattern>
+                </relocation>
                 <relocation>
                   <pattern>io.grpc</pattern>
                   <shadedPattern>com.google.bigtable.repackaged.io.grpc</shadedPattern>
@@ -183,16 +191,12 @@ limitations under the License.
                     https://github.com/grpc/grpc-java/pull/2485
                 -->
                 <relocation>
-                  <pattern>io.netty</pattern>
-                  <shadedPattern>com.google.bigtable.repackaged.io.netty</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>META-INF/native/libnetty</pattern>
-                  <shadedPattern>META-INF/native/libcom_google_bigtable_repackaged_netty</shadedPattern>
+                  <pattern>META-INF/native/libio_grpc_netty_shaded_netty</pattern>
+                  <shadedPattern>META-INF/native/libcom_google_bigtable_repackaged_io_grpc_netty_shaded_netty</shadedPattern>
                 </relocation>
                 <relocation>
-                  <pattern>META-INF/native/netty</pattern>
-                  <shadedPattern>META-INF/native/com_google_bigtable_repackaged_netty</shadedPattern>
+                  <pattern>META-INF/native/io_grpc_netty_shaded_netty</pattern>
+                  <shadedPattern>META-INF/native/com_google_bigtable_repackaged_io_grpc_netty_shaded_netty</shadedPattern>
                 </relocation>
 
                 <relocation>
diff --git a/pom.xml b/pom.xml
index d91d9f8b5..5d0d53b90 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,8 +57,6 @@ limitations under the License.
         <google-cloud-bom.version>0.50.0-alpha</google-cloud-bom.version>
         <grpc.version>1.11.0</grpc.version>
         <opencensus.version>0.13.0</opencensus.version>
-        <netty.version>4.1.22.Final</netty.version>
-        <netty-tcnative-boringssl-static.version>2.0.7.Final</netty-tcnative-boringssl-static.version>
         <protobuff-java.version>3.5.1</protobuff-java.version>
         <google.api.client.version>1.23.0</google.api.client.version>
         <google.http.client.version>1.23.0</google.http.client.version>


Test patch:
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableSession.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableSession.java
index dc0540d10..38f00621b 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableSession.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableSession.java
@@ -17,19 +17,15 @@
 package com.google.cloud.bigtable.grpc;
 
 import java.io.IOException;
-import java.lang.reflect.Field;
 
 import com.google.cloud.bigtable.config.CredentialOptions;
-import org.junit.Assert;
+import io.grpc.netty.shaded.io.netty.handler.ssl.OpenSsl;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
 import com.google.cloud.bigtable.config.BigtableOptions;
 
-import io.netty.handler.ssl.OpenSsl;
-import io.netty.util.Recycler;
-
 @SuppressWarnings({"resource","unused"})
 public class TestBigtableSession {
 
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestBulkMutationAwaitCompletion.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestBulkMutationAwaitCompletion.java
index 4a10dbded..e762442cc 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestBulkMutationAwaitCompletion.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestBulkMutationAwaitCompletion.java
@@ -26,12 +26,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -57,7 +52,6 @@ import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
 import com.google.rpc.Status;
 
-import io.netty.util.concurrent.ScheduledFuture;
 
 /**
  * Tests for {@link BulkMutation} that ensure that RPC failures and highly asynchronous calls work
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
index d14fb893e..55e360f4c 100644
--- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
+++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
@@ -28,7 +28,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
-import com.google.bigtable.repackaged.io.netty.handler.ssl.OpenSsl;
+import com.google.bigtable.repackaged.io.grpc.netty.shaded.io.netty.handler.ssl.OpenSsl;
 import com.google.cloud.bigtable.hbase2_x.BigtableConnection;
 
 
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
index c524f82f6..893753e51 100644
--- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
+++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
@@ -15,6 +15,7 @@
  */
 package com.google.cloud.bigtable.hbase;
 
+import io.netty.handler.ssl.OpenSsl;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
@@ -40,7 +41,7 @@ public class TestBigtableConnection {
   public void testBigtableConnectionExists() {
     Assert.assertEquals(BigtableConnection.class, BigtableConfiguration.getConnectionClass());
   }
-
+  
   @Test
   public void testTable() throws IOException {
     Configuration conf = BigtableConfiguration.configure("projectId", "instanceId");
@@ -51,4 +52,11 @@ public class TestBigtableConnection {
     Table table = connection.getTable(TableName.valueOf("someTable"));
     BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf("someTable"));
   }
+
+  @Test
+  public void testOpenSSL() throws Throwable{
+    if(!OpenSsl.isAvailable()){
+      throw OpenSsl.unavailabilityCause();
+    }
+  }
 }
diff --git a/bigtable-hbase-parent/bigtable-hbase-1.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java b/bigtable-hbase-parent/bigtable-hbase-1.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
index b55052a8a..d97f98a77 100644
--- a/bigtable-hbase-parent/bigtable-hbase-1.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
+++ b/bigtable-hbase-parent/bigtable-hbase-1.x-hadoop/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
@@ -15,14 +15,20 @@
  */
 package com.google.cloud.bigtable.hbase;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Table;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
-import com.google.bigtable.repackaged.io.netty.handler.ssl.OpenSsl;
+import com.google.bigtable.repackaged.io.grpc.netty.shaded.io.netty.handler.ssl.OpenSsl;
 import com.google.cloud.bigtable.hbase1_x.BigtableConnection;
 
+import java.io.IOException;
 
 /**
  * This is a test to ensure that BigtableConnection can find {@link BigtableConnection}
@@ -37,11 +43,21 @@ public class TestBigtableConnection {
   }
 
 
+  @Test
+  public void testTable() throws IOException {
+    Configuration conf = BigtableConfiguration.configure("projectId", "instanceId");
+    conf.set(BigtableOptionsFactory.BIGTABLE_NULL_CREDENTIAL_ENABLE_KEY, "true");
+    conf.set(BigtableOptionsFactory.BIGTABLE_USE_SERVICE_ACCOUNTS_KEY, "false");
+    BigtableConnection connection = new BigtableConnection(conf);
+    Admin admin = connection.getAdmin() ;
+    Table table = connection.getTable(TableName.valueOf("someTable"));
+    BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf("someTable"));
+  }
+
   @Test
   public void testOpenSSL() throws Throwable{
     if(!OpenSsl.isAvailable()){
       throw OpenSsl.unavailabilityCause();
     }
   }
-
 }
diff --git a/bigtable-hbase-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java b/bigtable-hbase-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
index 6e11aa843..3fb48d09f 100644
--- a/bigtable-hbase-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
+++ b/bigtable-hbase-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java
@@ -15,6 +15,12 @@
  */
 package com.google.cloud.bigtable.hbase;
 
+import io.netty.handler.ssl.OpenSsl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Table;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -22,6 +28,8 @@ import org.junit.runners.JUnit4;
 
 import com.google.cloud.bigtable.hbase1_x.BigtableConnection;
 
+import java.io.IOException;
+
 /**
  * This is a test to ensure that BigtableConnection can find {@link BigtableConnection}
  *
@@ -33,4 +41,22 @@ public class TestBigtableConnection {
   public void testBigtableConnectionExists() {
     Assert.assertEquals(BigtableConnection.class, BigtableConfiguration.getConnectionClass());
   }
+
+  @Test
+  public void testTable() throws IOException {
+    Configuration conf = BigtableConfiguration.configure("projectId", "instanceId");
+    conf.set(BigtableOptionsFactory.BIGTABLE_NULL_CREDENTIAL_ENABLE_KEY, "true");
+    conf.set(BigtableOptionsFactory.BIGTABLE_USE_SERVICE_ACCOUNTS_KEY, "false");
+    BigtableConnection connection = new BigtableConnection(conf);
+    Admin admin = connection.getAdmin() ;
+    Table table = connection.getTable(TableName.valueOf("someTable"));
+    BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf("someTable"));
+  }
+
+  @Test
+  public void testOpenSSL() throws Throwable{
+    if(!OpenSsl.isAvailable()){
+      throw OpenSsl.unavailabilityCause();
+    }
+  }
 }