Repository: jtablesaw/tablesaw
Pull request #265 (2018-04-06 11:26:59)

Correlation
Thanks for contributing.

- [x] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

Added support for 3 correlation calculations: Pearsons, Spearman, Kendall. 
## Testing

Tests were added for each


Patch:
diff --git a/core/src/main/java/tech/tablesaw/api/NumberColumn.java b/core/src/main/java/tech/tablesaw/api/NumberColumn.java
index 07f07d2a..ae6ce246 100644
--- a/core/src/main/java/tech/tablesaw/api/NumberColumn.java
+++ b/core/src/main/java/tech/tablesaw/api/NumberColumn.java
@@ -242,10 +242,6 @@ public class NumberColumn extends AbstractColumn implements DoubleIterable, IntC
         return column;
     }
 
-    public DoubleArrayList data() {
-        return data;
-    }
-
     public double firstElement() {
         if (size() > 0) {
             return data.getDouble(0);
diff --git a/core/src/main/java/tech/tablesaw/columns/numbers/NumberMapUtils.java b/core/src/main/java/tech/tablesaw/columns/numbers/NumberMapUtils.java
index b2d7b7fd..e83dcf92 100644
--- a/core/src/main/java/tech/tablesaw/columns/numbers/NumberMapUtils.java
+++ b/core/src/main/java/tech/tablesaw/columns/numbers/NumberMapUtils.java
@@ -370,6 +370,5 @@ public interface NumberMapUtils extends Column, NumberReduceUtils, DoubleIterabl
         return newColumn;
     }
 
-
     double get(int i);
 }
diff --git a/core/src/main/java/tech/tablesaw/columns/numbers/NumberReduceUtils.java b/core/src/main/java/tech/tablesaw/columns/numbers/NumberReduceUtils.java
index f73a78bd..881f8ef5 100644
--- a/core/src/main/java/tech/tablesaw/columns/numbers/NumberReduceUtils.java
+++ b/core/src/main/java/tech/tablesaw/columns/numbers/NumberReduceUtils.java
@@ -2,6 +2,10 @@ package tech.tablesaw.columns.numbers;
 
 import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
 import it.unimi.dsi.fastutil.doubles.DoubleSet;
+import org.apache.commons.math3.exception.NotANumberException;
+import org.apache.commons.math3.stat.correlation.KendallsCorrelation;
+import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
+import org.apache.commons.math3.stat.correlation.SpearmansCorrelation;
 import tech.tablesaw.aggregate.AggregateFunctions;
 import tech.tablesaw.api.NumberColumn;
 import tech.tablesaw.columns.Column;
@@ -26,7 +30,7 @@ public interface NumberReduceUtils extends Column {
 
     // Reduce functions applied to the whole column
     default double sum() {
-        return sum.agg(this.asDoubleArray());
+        return sum.agg(asDoubleArray());
     }
 
     default double product() {
@@ -38,70 +42,106 @@ public interface NumberReduceUtils extends Column {
     }
 
     default double median() {
-        return median.agg(this.asDoubleArray());
+        return median.agg(asDoubleArray());
     }
 
     default double quartile1() {
-        return quartile1.agg(this.asDoubleArray());
+        return quartile1.agg(asDoubleArray());
     }
 
     default double quartile3() {
-        return quartile3.agg(this.asDoubleArray());
+        return quartile3.agg(asDoubleArray());
     }
 
     default double percentile(double percentile) {
-        return AggregateFunctions.percentile(this.asDoubleArray(), percentile);
+        return AggregateFunctions.percentile(asDoubleArray(), percentile);
     }
 
     default double range() {
-        return range.agg(this.asDoubleArray());
+        return range.agg(asDoubleArray());
     }
 
     default double max() {
-        return max.agg(this.asDoubleArray());
+        return max.agg(asDoubleArray());
     }
 
     default double min() {
-        return min.agg(this.asDoubleArray());
+        return min.agg(asDoubleArray());
     }
 
     default double variance() {
-        return variance.agg(this.asDoubleArray());
+        return variance.agg(asDoubleArray());
     }
 
     default double populationVariance() {
-        return populationVariance.agg(this.asDoubleArray());
+        return populationVariance.agg(asDoubleArray());
     }
 
     default double standardDeviation() {
-        return stdDev.agg(this.asDoubleArray());
+        return stdDev.agg(asDoubleArray());
     }
 
     default double sumOfLogs() {
-        return sumOfLogs.agg(this.asDoubleArray());
+        return sumOfLogs.agg(asDoubleArray());
     }
 
     default double sumOfSquares() {
-        return sumOfSquares.agg(this.asDoubleArray());
+        return sumOfSquares.agg(asDoubleArray());
     }
 
     default double geometricMean() {
-        return geometricMean.agg(this.asDoubleArray());
+        return geometricMean.agg(asDoubleArray());
     }
 
     /**
      * Returns the quadraticMean, aka the root-mean-square, for all values in this column
      */
     default double quadraticMean() {
-        return quadraticMean.agg(this.asDoubleArray());
+        return quadraticMean.agg(asDoubleArray());
     }
 
     default double kurtosis() {
-        return kurtosis.agg(this.asDoubleArray());
+        return kurtosis.agg(asDoubleArray());
     }
 
     default double skewness() {
-        return skewness.agg(this.asDoubleArray());
+        return skewness.agg(asDoubleArray());
+    }
+
+    /**
+     * Returns the pearson's correlation between the receiver and the otherColumn
+     **/
+    default double pearsons(NumberColumn otherColumn) {
+
+        double[] x = asDoubleArray();
+        double[] y = otherColumn.asDoubleArray();
+
+        return new PearsonsCorrelation().correlation(x, y);
+    }
+
+    /**
+     * Returns the Spearman's Rank correlation between the receiver and the otherColumn
+     * @param otherColumn  A NumberColumn with no missing values
+     * @throws NotANumberException if either column contains any missing values
+     *
+     **/
+    default double spearmans(NumberColumn otherColumn) {
+
+        double[] x = asDoubleArray();
+        double[] y = otherColumn.asDoubleArray();
+
+        return new SpearmansCorrelation().correlation(x, y);
+    }
+
+    /**
+     * Returns the Kendall's Tau Rank correlation between the receiver and the otherColumn
+     **/
+    default double kendalls(NumberColumn otherColumn) {
+
+        double[] x = asDoubleArray();
+        double[] y = otherColumn.asDoubleArray();
+
+        return new KendallsCorrelation().correlation(x, y);
     }
 
     /**


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java b/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java
index ebd0f769..fe162c8a 100644
--- a/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java
+++ b/core/src/test/java/tech/tablesaw/api/NumberColumnTest.java
@@ -17,6 +17,9 @@ package tech.tablesaw.api;
 import com.google.common.base.Stopwatch;
 import io.codearte.jfairy.Fairy;
 import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
+import org.apache.commons.math3.stat.correlation.KendallsCorrelation;
+import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
+import org.apache.commons.math3.stat.correlation.SpearmansCorrelation;
 import tech.tablesaw.columns.numbers.NumberColumnFormatter;
 import tech.tablesaw.filtering.Filter;
 import tech.tablesaw.selection.Selection;
@@ -32,6 +35,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import static java.lang.Double.NaN;
 import static org.junit.Assert.*;
 
 /**
@@ -128,6 +132,36 @@ public class NumberColumnTest {
         assertNotNull(result);
     }
 
+    @Test
+    public void testCorrelation() {
+        double[] x = new double[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+        double[] y = new double[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+
+        NumberColumn xCol = NumberColumn.create("x", x);
+        NumberColumn yCol = NumberColumn.create("y", y);
+
+        double resultP = xCol.pearsons(yCol);
+        double resultS = xCol.spearmans(yCol);
+        double resultK = xCol.kendalls(yCol);
+        assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
+        assertEquals(new SpearmansCorrelation().correlation(x, y), resultS, 0.0001);
+        assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
+    }
+
+    @Test
+    public void testCorrelation2() {
+        double[] x = new double[]{1, 2, 3, 4, 5, 6, 7, NaN, 9, 10};
+        double[] y = new double[]{1, 2, 3, NaN, 5, 6, 7, 8, 9, 10};
+
+        NumberColumn xCol = NumberColumn.create("x", x);
+        NumberColumn yCol = NumberColumn.create("y", y);
+
+        double resultP = xCol.pearsons(yCol);
+        double resultK = xCol.kendalls(yCol);
+        assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
+        assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
+    }
+
     @Test
     public void testBetweenExclusive() {
         int[] originalValues = new int[]{32, 42, 40, 57, 52, -2};
diff --git a/core/src/test/java/tech/tablesaw/selection/SelectionTest.java b/core/src/test/java/tech/tablesaw/selection/SelectionTest.java
index 6c25bb64..e59ec846 100644
--- a/core/src/test/java/tech/tablesaw/selection/SelectionTest.java
+++ b/core/src/test/java/tech/tablesaw/selection/SelectionTest.java
@@ -47,7 +47,7 @@ public class SelectionTest {
         int[] expected = {42, 53, 111};
         assertArrayEquals(expected, selection.toArray());
     }
-    
+
     @Test
     public void add() {
         Selection selection = Selection.with(42, 53, 111);