GitHub PullRequestReviewCommentEvent at 2014-01-03 01:55:52
Repository: datafu
Actor: matthayes
Payload: {"comment":{"url":"https://api.github.com/repos/linkedin/datafu/pulls/comments/8628500","id":8628500,"diff_hunk":"@@ -0,0 +1,290 @@\n+/*\n+ * Copyright 2013 LinkedIn Corp. and contributors\n+ * \n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n+ * use this file except in compliance with the License. You may obtain a copy of\n+ * the License at\n+ * \n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ * \n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n+ * License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+\n+package datafu.pig.stats.entropy.stream;\n+\n+import org.apache.pig.AccumulatorEvalFunc;\n+\n+import java.io.IOException;\n+import java.util.List;\n+\n+import org.apache.pig.AccumulatorEvalFunc;\n+import org.apache.pig.builtin.Nondeterministic;\n+import org.apache.pig.data.DataBag;\n+import org.apache.pig.data.DataType;\n+import org.apache.pig.data.Tuple;\n+import org.apache.pig.impl.logicalLayer.FrontendException;\n+import org.apache.pig.backend.executionengine.ExecException;\n+import org.apache.pig.impl.logicalLayer.schema.Schema;\n+\n+import datafu.pig.stats.entropy.EntropyUtil;\n+\n+\n+/**\n+ * Calculate the conditional entropy H(Y|X) of random variables X and Y according to its \n+ * {@link <a href=\"http://en.wikipedia.org/wiki/Conditional_entropy\" target=\"_blank\">wiki definition</a>}, \n+ * X is the conditional variable and Y is the variable that conditions on X.\n+ * <p>\n+ * The input to this UDF is a bag of 2 field tuple.\n+ * <ul>\n+ *     <li>the 1st field of the tuple is an instance of variable X.\n+ *     <li>the 2nd field of the tuple is an instance of variable Y.\n+ * </ul>\n+ * </p>\n+ * <p>\n+ * An exception will be thrown if the input tuple does not have 2 fields.\n+ * </p> \n+ * \n+ * <p>\n+ * This UDF's constructor definition and parameters are the same as that of * {@link datafu.pig.stats.entropy.stream.StreamingEntropy}\n+ * </p>\n+ *\n+ * <p>\n+ * Note:\n+ * <ul>\n+ *     <li>input bag to the UDF must be sorted on X and Y\n+ *     <li>Entropy value is returned as double type.\n+ * </ul>\n+ * </p>\n+ *\n+ * <p>\n+ * How to use: \n+ * </p>\n+ * <p>\n+ * This UDF is suitable to calculate conditional entropy in a nested FOREACH after a GROUP BY,\n+ * where we sort the inner bag and use the sorted bag as the input to this UDF.\n+ * </p>\n+ * <p>\n+ * This is a scenario in which we would like to get a variable's conditional entropy in different constraint groups.\n+ * </p>\n+ * <p>\n+ * Example:\n+ * <pre>\n+ * \n+ * {@code\n+ * \n+ * --define empirical conditional entropy with Euler's number as the logarithm base\n+ * define CondEntropy datafu.pig.stats.entropy.stream.StreamingCondEntropy();\n+ *\n+ * input = LOAD 'input' AS (grp: chararray, valX: double, valY: double);\n+ *\n+ * -- calculate conditional entropy H(Y|X) in each group\n+ * input_group_g = GROUP input BY grp;\n+ * entropy_group = FOREACH input_group_g {\n+ *   input_val = input.(valX, valY)\n+ *   input_ordered = ORDER input_val BY $0, $1;\n+ *   GENERATE FLATTEN(group) AS group, CondEntropy(input_ordered) AS cond_entropy; \n+ * }\n+ * }\n+ * </pre>\n+ * </p>\n+ * <p>\n+ * Use case to calculate mutual information:\n+ * <pre>\n+ * {@code\n+ * ------------\n+ * -- calculate mutual information I(X, Y) using streaming conditional entropy and streaming entropy\n+ * -- I(X, Y) = H(Y) - H(Y|X)\n+ * ------------\n+ * \n+ * define CondEntropy datafu.pig.stats.entropy.stream.StreamingCondEntropy();\n+ * define Entropy datafu.pig.stats.entropy.stream.StreamingEntropy();\n+ * \n+ * input = LOAD 'input' AS (grp: chararray, valX: double, valY: double);\n+ * \n+ * -- calculate the I(X,Y) in each group\n+ * input_group_g = GROUP input BY grp;\n+ * mutual_information = FOREACH input_group_g {\n+ *      input_val_x_y = input.(valX, valY);\n+ *      input_val_x_y_ordered = ORDER input_val_x_y BY $0,$1;\n+ *      input_val_y = input.valY;\n+ *      input_val_y_ordered = ORDER input_val_y BY $0;\n+ *      cond_h_x_y = CondEntropy(input_val_x_y_ordered);\n+ *      h_y = Entropy(input_val_y_ordered);\n+ *      GENERATE FLATTEN(group), h_y - cond_h_x_y;\n+ * }\n+ * }\n+ * </pre>\n+ * </p>\n+ * \n+ * @see StreamingEntropy\n+ */\n+@Nondeterministic\n+public class StreamingCondEntropy extends AccumulatorEvalFunc<Double> {\n+    //last visited tuple of <x,y>\n+    private Tuple xy;\n+    \n+    //number of occurrence of last visited <x,y>\n+    private long cxy;\n+    \n+    //number of occurrence of last visited x\n+    private long cx;\n+    \n+    //comparison result between the present tuple and the last visited tuple\n+    private int lastCmp;\n+    \n+    //entropy estimator for H(x,y)\n+    private EntropyEstimator combEstimator;\n+    \n+    //entropy estimator for H(x)\n+    private EntropyEstimator condXEstimator;\n+    \n+    public StreamingCondEntropy() throws ExecException\n+    {\n+      this(EntropyEstimator.EMPIRICAL_ESTIMATOR);\n+    }\n+    \n+    public StreamingCondEntropy(String type) throws ExecException \n+    {\n+      this(type, EntropyUtil.LOG);\n+    }\n+\n+    public StreamingCondEntropy(String type, String base) throws ExecException\n+    {\n+      try {\n+          this.combEstimator = EntropyEstimator.createEstimator(type, base);\n+          this.condXEstimator = EntropyEstimator.createEstimator(type, base);\n+      } catch (IllegalArgumentException ex) {\n+          throw new ExecException(String.format(\n+                  \"Fail to initialize StreamingCondEntropy with entropy estimator of type (%s), base: (%s). Exception: (%s)\",\n+                  type, base, ex)); \n+      }\n+      cleanup();\n+    }\n+    \n+    /*\n+     * Accumulate occurrence frequency of <x,y> and x\n+     * as we stream through the input bag\n+     */\n+    @Override\n+    public void accumulate(Tuple input) throws IOException\n+    {\n+      for (Tuple t : (DataBag) input.get(0)) {\n+","path":"src/java/datafu/pig/stats/entropy/stream/StreamingCondEntropy.java","position":176,"original_position":176,"commit_id":"eac3cedc6ac9a91f7b87c642a495a6e27da04fd1","original_commit_id":"eac3cedc6ac9a91f7b87c642a495a6e27da04fd1","user":{"login":"matthayes","id":212331,"avatar_url":"https://gravatar.com/avatar/d8c501c3eb6c504475bc945c8972ab52?d=https%3A%2F%2Fidenticons.github.com%2Fff788f50a9c8aae646ff817e78f3eef9.png&r=x","gravatar_id":"d8c501c3eb6c504475bc945c8972ab52","url":"https://api.github.com/users/matthayes","html_url":"https://github.com/matthayes","followers_url":"https://api.github.com/users/matthayes/followers","following_url":"https://api.github.com/users/matthayes/following{/other_user}","gists_url":"https://api.github.com/users/matthayes/gists{/gist_id}","starred_url":"https://api.github.com/users/matthayes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthayes/subscriptions","organizations_url":"https://api.github.com/users/matthayes/orgs","repos_url":"https://api.github.com/users/matthayes/repos","events_url":"https://api.github.com/users/matthayes/events{/privacy}","received_events_url":"https://api.github.com/users/matthayes/received_events","type":"User","site_admin":false},"body":"yea, true.  i was thinking it's worth as an added safety check, given that the tuple may not necessarily be consistent with the reported schema (e.g. as example of poorly written UDF producing input data).  maybe this is overkill though.","created_at":"2014-01-03T01:55:52Z","updated_at":"2014-01-03T01:55:52Z","html_url":"https://github.com/linkedin/datafu/pull/78#discussion_r8628500","pull_request_url":"https://api.github.com/repos/linkedin/datafu/pulls/78","_links":{"self":{"href":"https://api.github.com/repos/linkedin/datafu/pulls/comments/8628500"},"html":{"href":"https://github.com/linkedin/datafu/pull/78#discussion_r8628500"},"pull_request":{"href":"https://api.github.com/repos/linkedin/datafu/pulls/78"}}}}