Repository: Vantiq/vantiq-extension-sources (language: python)

Issue:
Camel Connector: Allow properties to require RAW handling
When providing RAW values, it is sometime necessary to wrap them in RAW indicators -- this tells the Camel mechanism to alter any URL encoding going on.  For some cases, we'd like to have the connector mechanism handle this, so add handling for a new config property which is a set of lists of property names that require RAW handling.  Upon receipt of the configuration, these values will be wrapped in RAW indicators before being given to Camel's Property component.


Discussion / hints:

Fix patch:
diff --git a/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/connector/CamelHandleConfiguration.java b/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/connector/CamelHandleConfiguration.java
index e5643b26..6b18b763 100644
--- a/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/connector/CamelHandleConfiguration.java
+++ b/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/connector/CamelHandleConfiguration.java
@@ -61,9 +61,17 @@ public class CamelHandleConfiguration extends Handler<ExtensionServiceMessage> {
     public static final String ROUTES_FORMAT = "routesFormat";
     public static final String COMPONENT_PROPERTIES = "componentProperties";
     public static final String PROPERTY_VALUES = "propertyValues";
+    public static final String RAW_REQUIRED = "rawValuesRequired";
+    public static final String DISCOVERED_RAW = "discovered";
+    public static final String CONFIGURED_RAW = "configured";
     
-    public static final String VANTIQ = "vantiq";
+    public static final String NO_RAW_REQUEST = "!NORAW!";
+    public static final String RAW_START = "RAW(";
+    public static final String RAW_END = ")";
+    public static final String RAW_START_ALT = "RAW{";
+    public static final String RAW_END_ALT = "}";
     
+    public static final String VANTIQ = "vantiq";
     public static final String GENERAL = "general";
     public static final String COMPONENT_CACHE = "componentCacheDirectory";
     public static final String COMPONENT_CACHE_DEFAULT = "componentCache";
@@ -144,25 +152,15 @@ public void handleMessage(ExtensionServiceMessage message) {
                           PROPERTY_VALUES, target.getClass().getName());
                 failConfig();
                 return;
-            } else {
-                // It's coming from JSON, so the key must be a string...
-                Map<String, Object> input = (Map<String, Object>) target;
-                input.forEach( (k, v) -> {
-                    if (v == null) {
-                        String kclass = "null";
-                        String vclass = "null";
-                        if (k != null) {
-                            kclass = k.getClass().getName();
-                        }
-                        if (v != null && v.getClass() != null) {
-                            vclass = v.getClass().getName();
-                        }
-                        log.error("Camel connector {} value must have a String key ({}: {})  and value ({}: {}).",
-                                  PROPERTY_VALUES, k, kclass, v, vclass);
-                        failConfig();
-                    }
-                });
-                
+            }
+        }
+        
+        target = camelConfig.get(RAW_REQUIRED);
+        if (target != null) {
+            if (!(target instanceof Map)) {
+                log.error("Camel connector property {} should be a single JSON object (found {}).",
+                          RAW_REQUIRED, target);
+                failConfig();
             }
         }
         
@@ -258,12 +256,83 @@ public boolean runCamelApp(Map<String, Object> camelConfig, Map<String, Object>
             // This is checked in the caller
             //noinspection unchecked
             Map<String, Object> input = (Map<String, Object>) camelConfig.get(PROPERTY_VALUES);
+            //noinspection unchecked
+            Map<String, List<String>> raw = (Map<String, List<String>>) camelConfig.get(RAW_REQUIRED);
+            List<String> rawRequired = new ArrayList<>();
+            // Create a list from the various categories that may be in this set...
+            if (raw != null) {
+                raw.forEach((k, v) -> {
+                    rawRequired.addAll(v);
+                });
+            }
             // Camel wants these as a Java Properties object, so perform the conversion as required.
             Properties propVals = null;
             if (input != null) {
                 propVals = new Properties(input.size());
                 for (Map.Entry<String, Object> p : input.entrySet()) {
-                    propVals.setProperty(p.getKey(), p.getValue().toString());
+                    if (p.getValue() != null) {
+                        // Don't set property values to null.
+                        
+                        Object val = p.getValue();
+                        boolean passThru = true; // By default, pass things thru
+                        String startRaw = RAW_START;
+                        String endRaw = RAW_END;
+    
+                        // Some properties (typically, but not necessarily, credentials of some sort) may need to be
+                        // passed as Camel RAW-wrapped values.  This affects how the Camel system will pass these
+                        // values to the property consumers involved. If the source configuration says that this
+                        // property should be RAW wrapped, adjust as required.
+                        if (rawRequired.contains(p.getKey())) {
+                            boolean isString = val instanceof String;
+                            if (isString) {
+                                String valStr = (String) val;
+                                boolean suppressed = valStr.startsWith(NO_RAW_REQUEST);
+                                if (suppressed) {
+                                    valStr = valStr.substring(NO_RAW_REQUEST.length());
+                                } else {
+                                    // If the caller has already wrapped the value (RAW() or RAW{}), then pass it
+                                    // thru unchanged.  Similarly, if the value contains both end characters, we pass
+                                    // it thru.  After seeing the start string, Camel will scan for the corresponding
+                                    // end character. However, if both end characters are present, either start
+                                    // sequence will terminate prematurely. As a consequence, attempting to wrap
+                                    // things as RAW strings will not work correctly, so we pass the string thru
+                                    // unchanged.
+                                    //
+                                    // Our other choice here would be to signal an error. As a layer atop the Camel
+                                    // system, any decision to fail in this situation is not ours to make. The code
+                                    // that's using this property value may have handling present -- we don't know.
+                                    // Moreover, the request to RAW-wrap these property values may is provided by the
+                                    // source configuration which may be part of some assembly, so that configuration
+                                    // should not prevent the callers request from being passed along.
+                                    // It is better to choose the route of less interference, letting the
+                                    // underlying systems do their work. This will succeed or fail as would happen
+                                    // without our automatic wrapping here, so we'll let the underlying systems
+                                    // report things as they will, and let the caller work things out, should that
+                                    // working out be necessary.
+                                    
+                                    // passThru set to true if both end characters or if the value is already wrapped.
+                                    passThru = (valStr.contains(RAW_END) && valStr.contains(RAW_END_ALT)) ||
+                                            (valStr.startsWith(RAW_START) && valStr.endsWith(RAW_END)) ||
+                                            (valStr.startsWith(RAW_START_ALT) && valStr.endsWith(RAW_END_ALT));
+                                    if (!passThru) {
+                                        // If we're wrapping, do we need to use the alternate wrapping
+                                        if (valStr.contains(RAW_END)) {
+                                            startRaw = RAW_START_ALT;
+                                            endRaw = RAW_END_ALT;
+                                        }
+                                    }
+                                }
+                                // our string may have been altered, so set the value to send
+                                val = valStr;
+                            }
+                        }
+                        if (passThru) {
+                            // If we're suppressing or we have a value with both ) & }, leave things alone
+                            propVals.setProperty(p.getKey(), val.toString());
+                        } else {
+                            propVals.setProperty(p.getKey(), startRaw + val + endRaw);
+                        }
+                    }
                 }
             }
             log.debug("Properties provided: {}", propVals);
diff --git a/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/discover/CamelRunner.java b/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/discover/CamelRunner.java
index 93174876..09fbe96e 100644
--- a/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/discover/CamelRunner.java
+++ b/camelConnector/src/main/java/io/vantiq/extsrc/camelconn/discover/CamelRunner.java
@@ -143,6 +143,10 @@ protected void doInit() throws Exception {
         createCamelContext();
     }
     
+    public Properties getCamelProperties() {
+        return camelProperties;
+    }
+    
     /**
      * Build the application classloader we will use.
      *
@@ -397,7 +401,7 @@ protected void beforeStart() throws Exception {
                     }
                 }
             }
-    
+
             // Some routes may have components (e.g., Salesforce) that need specific configuration.
             // If that's the case, do that now before we start our route(s).
             if (initComponents != null && initComponents.size() > 0) {


Test patch:
diff --git a/camelConnector/src/test/java/io/vantiq/extsrc/camelconn/connector/VantiqConfigurationTest.java b/camelConnector/src/test/java/io/vantiq/extsrc/camelconn/connector/VantiqConfigurationTest.java
index dcfd1af5..3b0b38ab 100644
--- a/camelConnector/src/test/java/io/vantiq/extsrc/camelconn/connector/VantiqConfigurationTest.java
+++ b/camelConnector/src/test/java/io/vantiq/extsrc/camelconn/connector/VantiqConfigurationTest.java
@@ -1,5 +1,11 @@
 package io.vantiq.extsrc.camelconn.connector;
 
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.NO_RAW_REQUEST;
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.RAW_END;
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.RAW_END_ALT;
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.RAW_START;
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.RAW_START_ALT;
+
 import static io.vantiq.extjsdk.ExtensionServiceMessage.OP_CONFIGURE_EXTENSION;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.APP_NAME;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.CAMEL_APP;
@@ -7,8 +13,10 @@
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.COMPONENT_CACHE;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.COMPONENT_LIB;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.COMPONENT_PROPERTIES;
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.DISCOVERED_RAW;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.GENERAL;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.PROPERTY_VALUES;
+import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.RAW_REQUIRED;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.ROUTES_FORMAT;
 import static io.vantiq.extsrc.camelconn.connector.CamelHandleConfiguration.ROUTES_LIST;
 import static io.vantiq.extsrc.camelconn.discover.VantiqComponentResolverTest.MISSING_VALUE;
@@ -52,13 +60,19 @@ public class VantiqConfigurationTest {
     
     // Interface to use in declaration.  We'll pass lambda's in to do the actual verification work
     interface Verifier {
-        void doVerify(CamelContext runnerContext);
+        void doVerify(CamelContext runnerContext, Properties props);
     }
     
     void performConfigTest(String appName, String route, String routeFormat,
                            List<Map<String, Object>> compInitProps, Properties propertyValues, Verifier vfy) {
+        performConfigTest(appName, route, routeFormat, compInitProps, propertyValues, vfy, null);
+    }
+    void performConfigTest(String appName, String route, String routeFormat,
+                           List<Map<String, Object>> compInitProps, Properties propertyValues, Verifier vfy,
+                           List<String> rawReq) {
         assumeTrue(!sfLoginUrl.equals(MISSING_VALUE) && !sfClientId.equals(MISSING_VALUE) &&
                            !sfClientSecret.equals(MISSING_VALUE) && !sfRefreshToken.equals(MISSING_VALUE));
+        
         Map<String, Object> simpleConfig = new HashMap<>();
         Map<String, Object> camelConfig = new HashMap<>();
         simpleConfig.put(CAMEL_CONFIG, camelConfig);
@@ -66,10 +80,10 @@ void performConfigTest(String appName, String route, String routeFormat,
         camelConfig.put(CAMEL_APP, camelAppConfig);
         Map<String, String> generalConfig = new HashMap<>();
         camelConfig.put(GENERAL, generalConfig);
-        
+
         generalConfig.put(COMPONENT_CACHE, CACHE_DIR);
         generalConfig.put(COMPONENT_LIB, LOADED_LIBRARIES);
-        
+
         camelAppConfig.put(ROUTES_LIST, route);
         camelAppConfig.put(ROUTES_FORMAT, routeFormat);
         camelAppConfig.put(APP_NAME, appName);
@@ -79,16 +93,22 @@ void performConfigTest(String appName, String route, String routeFormat,
         if (propertyValues != null) {
             camelAppConfig.put(PROPERTY_VALUES, propertyValues);
         }
-        
+        if (rawReq != null) {
+            Map<String, Object> rawReqMap = new HashMap<>();
+            
+            rawReqMap.put(DISCOVERED_RAW, rawReq);
+            camelAppConfig.put(RAW_REQUIRED, rawReqMap);
+        }
+
         String fauxVantiqUrl = "http://someVantiqServer";
         ExtensionServiceMessage esm = new ExtensionServiceMessage(fauxVantiqUrl);
         esm.op = OP_CONFIGURE_EXTENSION;
         esm.object = simpleConfig;
-        
+
         CamelCore core = new CamelCore("testComponentInitConfiguration",
                                        "someAccessToken", fauxVantiqUrl);
         CamelHandleConfiguration handler = new CamelHandleConfiguration(core);
-        
+
         handler.handleMessage(esm);
         assertTrue("handler completed", handler.isComplete());
         assertNotNull("Camel Runner", handler.getCurrentCamelRunner());
@@ -100,8 +120,12 @@ void performConfigTest(String appName, String route, String routeFormat,
         
         CamelContext runnerContext = handler.getCurrentCamelRunner().getCamelContext();
         assertTrue("Context Running", runnerContext.isStarted());
-        
-        vfy.doVerify(runnerContext);
+
+        try {
+            vfy.doVerify(runnerContext, handler.getCurrentCamelRunner().getCamelProperties());
+        } finally {
+            handler.getCurrentCamelRunner().close();
+        }
     
         handler.getCurrentCamelRunner().close();
         try {
@@ -111,11 +135,185 @@ void performConfigTest(String appName, String route, String routeFormat,
         }
     }
     
+    public static final String ROUTE_NOT_USED = ""
+            + "- route:\n"
+            + "    id: \"YAML route just there for existence\"\n"
+            + "    from:\n"
+            + "      uri: \"direct:start\"\n"
+            + "      steps:\n"
+            + "        - to:\n"
+            + "            uri: \"log:simplelog\"\n";
+    
+    @Test
+    public void testRawNoRaw() {
+        Map<String, Object> propValues = Map.of("prop1", "prop1Value",
+                                                "prop2", "prop2Value",
+                                                "prop3", "prop3Value");
+        Properties props = new Properties(propValues.size());
+        // Though officially frowned upon, Properties.putAll here from a Map<String, String> is safe as it cannot put
+        // non-String keys or values into the Properties base map.
+        props.putAll(propValues);
+    
+        performConfigTest(name.getMethodName(), ROUTE_NOT_USED, "yaml",
+                          null, props,
+                          (CamelContext runnerContext, Properties lclProps) -> {
+                                assert lclProps != null;
+                                assert propValues.size() == lclProps.size();
+                                propValues.forEach( (name, val) -> {
+                                    assert lclProps.containsKey(name);
+                                    assert lclProps.getProperty(name).equals(val);
+                              });
+                          });
+    }
+    
+    @Test
+    public void testRawAlready() {
+        Map<String, Object> propValues = Map.of("prop1", "prop1Value",
+                                                "prop2", "RAW{prop2Value}",
+                                                "prop3", "RAW(prop3Value)",
+                                                "prop4", "RAW{prop4}value",
+                                                "prop5", "RAW{prop5value)");
+        Properties props = new Properties(propValues.size());
+        // Though officially frowned upon, Properties.putAll here from a Map<String, String> is safe as it cannot put
+        // non-String keys or values into the Properties base map.
+        props.putAll(propValues);
+        List<String> raws = List.of("prop1", "prop2", "prop3", "prop4");
+        List<String> unchanged = List.of("prop2", "prop3", "prop5");
+    
+        performConfigTest(name.getMethodName(), ROUTE_NOT_USED, "yaml",
+                          null, props,
+                          (CamelContext runnerContext, Properties lclProps) -> {
+                              assert lclProps != null;
+                              assert propValues.size() == lclProps.size();
+                              propValues.forEach( (name, val) -> {
+                                  assert lclProps.containsKey(name);
+                                  if (unchanged.contains(name)) {
+                                      assert lclProps.getProperty(name).equals(val);
+                                  } else {
+                                      assert lclProps.getProperty(name).equals(CamelHandleConfiguration.RAW_START +
+                                                              val + CamelHandleConfiguration.RAW_END);
+                                  }
+                              });
+                          }, raws);
+    }
+    
+    @Test
+    public void testRawRequired() {
+        Map<String, Object> propValues = Map.of("prop1", "prop1Value",
+                                                "prop2", "prop2Value",
+                                                "prop3", "prop3Value");
+        Properties props = new Properties(propValues.size());
+        // Though officially frowned upon, Properties.putAll here from a Map<String, String> is safe as it cannot put
+        // non-String keys or values into the Properties base map.
+        props.putAll(propValues);
+        List<String> raws = List.of("prop1", "prop3");
+    
+        performConfigTest(name.getMethodName(), ROUTE_NOT_USED, "yaml",
+                          null, props,
+                          (CamelContext runnerContext, Properties lclProps) -> {
+                              assert lclProps != null;
+                              assert propValues.size() == lclProps.size();
+                              propValues.forEach( (name, val) -> {
+                                  assert lclProps.containsKey(name);
+                                  if (raws.contains(name)) {
+                                      assert lclProps.getProperty(name).equals(CamelHandleConfiguration.RAW_START +
+                                              val + CamelHandleConfiguration.RAW_END);
+                                  } else {
+                                      assert lclProps.getProperty(name).equals(val);
+                                  }
+                              });
+                          }, raws);
+    }
+    
+    @Test
+    public void testRawAltRequired() {
+        Map<String, Object> propValues = Map.of("prop1", "prop1)Value",
+                                                "prop2", "prop2Value",
+                                                "prop3", "prop3Value");
+        Properties props = new Properties(propValues.size());
+        // Though officially frowned upon, Properties.putAll here from a Map<String, String> is safe as it cannot put
+        // non-String keys or values into the Properties base map.
+        props.putAll(propValues);
+        List<String> raws = List.of("prop1", "prop3");
+        
+        performConfigTest(name.getMethodName(), ROUTE_NOT_USED, "yaml",
+                          null, props,
+                          (CamelContext runnerContext, Properties lclProps) -> {
+                              assert lclProps != null;
+                              assert propValues.size() == lclProps.size();
+                              propValues.forEach( (name, val) -> {
+                                  assert lclProps.containsKey(name);
+                                  if (name.equals("prop1")) {
+                                      assert lclProps.getProperty(name).equals(CamelHandleConfiguration.RAW_START_ALT +
+                                                                                       val + CamelHandleConfiguration.RAW_END_ALT);
+                                  } else if (name.equals("prop3")) {
+                                      assert lclProps.getProperty(name).equals(CamelHandleConfiguration.RAW_START +
+                                                                                       val + CamelHandleConfiguration.RAW_END);
+                                  } else {
+                                      assert lclProps.getProperty(name).equals(val);
+                                  }
+                              });
+                          }, raws);
+    }
+    @Test
+    public void testRawSuppressed() {
+        Map<String, Object> propValues = Map.of("prop1", "prop1Value",
+                                                "prop2", NO_RAW_REQUEST + "prop2Value",
+                                                "prop3", "prop3Value");
+        Properties props = new Properties(propValues.size());
+        // Though officially frowned upon, Properties.putAll here from a Map<String, String> is safe as it cannot put
+        // non-String keys or values into the Properties base map.
+        props.putAll(propValues);
+        List<String> raws = List.of("prop2");
+    
+        performConfigTest(name.getMethodName(), ROUTE_NOT_USED, "yaml",
+                          null, props,
+                          (CamelContext runnerContext, Properties lclProps) -> {
+                              assert lclProps != null;
+                              assert propValues.size() == lclProps.size();
+                              propValues.forEach( (name, val) -> {
+                                  assert lclProps.containsKey(name);
+                                  if (raws.contains(name)) {
+                                      String newVal = ((String) propValues.get(name))
+                                              .substring(NO_RAW_REQUEST.length());
+                                      assert lclProps.getProperty(name).equals(newVal);
+                                  }
+                              });
+                          }, raws);
+    }
+    
+    @Test
+    public void testCannotWrap() {
+        Map<String, Object> propValues = Map.of("prop1", "prop1Value",
+                                                "prop2", "prop2Value",
+                                                "prop3", "prop3}Value)Value");
+        Properties props = new Properties(propValues.size());
+        // Though officially frowned upon, Properties.putAll here from a Map<String, String> is safe as it cannot put
+        // non-String keys or values into the Properties base map.
+        props.putAll(propValues);
+        List<String> raws = List.of("prop1", "prop3");
+    
+        performConfigTest(name.getMethodName(), ROUTE_NOT_USED, "yaml",
+                          null, props,
+                          (CamelContext runnerContext, Properties lclProps) -> {
+                              assert lclProps != null;
+                              assert propValues.size() == lclProps.size();
+                              propValues.forEach( (name, val) -> {
+                                  assert lclProps.containsKey(name);
+                                  if (name.equals("prop1")) {
+                                      assert lclProps.getProperty(name).equals(RAW_START + val + RAW_END);
+                                  } else {
+                                      assert lclProps.getProperty(name).equals(val);
+                                  }
+                              });
+                          }, raws);
+    
+    }
     @Test
     public void testSimpleConfiguration() {
         performConfigTest(name.getMethodName(), XML_ROUTE, "xml",
                           null, null,
-                          (CamelContext runnerContext) -> {
+                          (CamelContext runnerContext, Properties props) -> {
                               TriFunction<CamelContext, String, Object, Boolean> verifyOperation =
                                       defineVerifyOperation();
                               assert verifyOperation.apply(runnerContext, QUERY_MONKEY, RESPONSE_MONKEY);
@@ -127,7 +325,7 @@ public void testSimpleConfiguration() {
     public void testComponentInitConfiguration() {
         performConfigTest(name.getMethodName(), SALESFORCETASKS_YAML, "yaml",
                           getComponentsToInit(), null,
-                          (CamelContext runnerContext) -> {
+                          (CamelContext runnerContext, Properties props) -> {
                               TriFunction<CamelContext, String, Object, Boolean> verifyOperation =
                                       defineVerifyOperation();
                             assert verifyOperation.apply(runnerContext, "not used", Map.of("done", true));
@@ -143,7 +341,7 @@ public void testComponentInitConfigurationWithPropertyValues() {
         props.putAll(pValues);
         performConfigTest(name.getMethodName(), PARAMETERIZED_SALESFORCE_ROUTE, "yaml",
                           getComponentsToInit(), props,
-                          (CamelContext runnerContext) -> {
+                          (CamelContext runnerContext, Properties pros) -> {
                               TriFunction<CamelContext, String, Object, Boolean> verifyOperation =
                                       defineVerifyOperation();
                               assert verifyOperation.apply(runnerContext, "not used", Map.of("done", true));