Repository: jtablesaw/tablesaw
Pull request #735 (2020-01-08 00:16:02)

Upgrade to Smile 2.0.0
Thanks for contributing.

- [X] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://developercertificate.org)

## Description

Upgrades to Smile 2.0

## Testing

Updated the existing unit tests. Deleted a few since there are less cases to cover now


Patch:
diff --git a/core/pom.xml b/core/pom.xml
index 8018f3a3..8969fb50 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -140,7 +140,7 @@
         <dependency>
             <groupId>com.github.haifengl</groupId>
             <artifactId>smile-data</artifactId>
-            <version>1.5.3</version>
+            <version>2.0.0</version>
             <optional>true</optional>
             <scope>provided</scope> <!-- because the shade plugin doesn't exclude optional dependencies -->
         </dependency>
@@ -181,7 +181,7 @@
         <dependency>
             <groupId>com.github.haifengl</groupId>
             <artifactId>smile-core</artifactId>
-            <version>1.5.3</version>
+            <version>2.0.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
diff --git a/core/src/main/java/tech/tablesaw/conversion/smile/SmileConverter.java b/core/src/main/java/tech/tablesaw/conversion/smile/SmileConverter.java
index c599b9a0..bbb8e469 100644
--- a/core/src/main/java/tech/tablesaw/conversion/smile/SmileConverter.java
+++ b/core/src/main/java/tech/tablesaw/conversion/smile/SmileConverter.java
@@ -1,16 +1,18 @@
 package tech.tablesaw.conversion.smile;
 
-import java.text.ParseException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import smile.data.Attribute;
-import smile.data.AttributeDataset;
-import smile.data.NominalAttribute;
-import smile.data.NumericAttribute;
+import smile.data.DataFrame;
+import smile.data.Tuple;
+import smile.data.type.DataType;
+import smile.data.type.DataTypes;
+import smile.data.type.StructField;
+import smile.data.type.StructType;
 import tech.tablesaw.api.ColumnType;
-import tech.tablesaw.api.NumericColumn;
-import tech.tablesaw.api.StringColumn;
+import tech.tablesaw.api.InstantColumn;
 import tech.tablesaw.columns.Column;
 import tech.tablesaw.table.Relation;
 
@@ -22,125 +24,61 @@ public class SmileConverter {
     this.table = table;
   }
 
-  /** Returns a dataset where the response column is numeric. E.g. to be used for a regression */
-  public AttributeDataset numericDataset(String responseColName) {
-    return dataset(
-        table.numberColumn(responseColName),
-        AttributeType.NUMERIC,
-        table.numericColumns().stream()
-            .filter(c -> !c.name().equals(responseColName))
-            .collect(Collectors.toList()));
+  public DataFrame toDataFrame() {
+    StructType schema =
+        DataTypes.struct(
+            table.columns().stream()
+                .map(col -> new StructField(col.name(), toSmileType(col.type())))
+                .collect(Collectors.toList()));
+    return toDataFrame(schema);
   }
 
-  /** Returns a dataset where the response column is numeric. E.g. to be used for a regression */
-  public AttributeDataset numericDataset(int responseColIndex, int... variablesColIndices) {
-    return dataset(
-        table.numberColumn(responseColIndex),
-        AttributeType.NUMERIC,
-        table.columns(variablesColIndices));
-  }
-
-  /** Returns a dataset where the response column is numeric. E.g. to be used for a regression */
-  public AttributeDataset numericDataset(String responseColName, String... variablesColNames) {
-    return dataset(
-        table.numberColumn(responseColName),
-        AttributeType.NUMERIC,
-        table.columns(variablesColNames));
-  }
-
-  /**
-   * Returns a dataset where the response column is nominal. E.g. to be used for a classification
-   */
-  public AttributeDataset nominalDataset(String responseColName) {
-    return dataset(
-        table.numberColumn(responseColName),
-        AttributeType.NOMINAL,
-        table.numericColumns().stream()
-            .filter(c -> !c.name().equals(responseColName))
-            .collect(Collectors.toList()));
-  }
-
-  /**
-   * Returns a dataset where the response column is nominal. E.g. to be used for a classification
-   */
-  public AttributeDataset nominalDataset(int responseColIndex, int... variablesColIndices) {
-    return dataset(
-        table.numberColumn(responseColIndex),
-        AttributeType.NOMINAL,
-        table.columns(variablesColIndices));
-  }
-
-  /**
-   * Returns a dataset where the response column is nominal. E.g. to be used for a classification
-   */
-  public AttributeDataset nominalDataset(String responseColName, String... variablesColNames) {
-    return dataset(
-        table.numberColumn(responseColName),
-        AttributeType.NOMINAL,
-        table.columns(variablesColNames));
-  }
-
-  private AttributeDataset dataset(
-      NumericColumn<?> responseCol, AttributeType type, List<Column<?>> variableCols) {
-    List<Column<?>> convertedVariableCols =
-        variableCols.stream()
-            .map(col -> col.type() == ColumnType.STRING ? col : table.nCol(col.name()))
-            .collect(Collectors.toList());
-    Attribute responseAttribute =
-        type == AttributeType.NOMINAL
-            ? colAsNominalAttribute(responseCol)
-            : new NumericAttribute(responseCol.name());
-    AttributeDataset dataset =
-        new AttributeDataset(
-            table.name(),
-            convertedVariableCols.stream()
-                .map(col -> colAsAttribute(col))
-                .toArray(Attribute[]::new),
-            responseAttribute);
-    for (int i = 0; i < responseCol.size(); i++) {
-      final int r = i;
-      double[] x =
-          IntStream.range(0, convertedVariableCols.size())
-              .mapToDouble(c -> getDouble(convertedVariableCols.get(c), dataset.attributes()[c], r))
-              .toArray();
-      dataset.add(x, responseCol.getDouble(r));
-    }
-    return dataset;
-  }
-
-  private double getDouble(Column<?> col, Attribute attr, int r) {
-    if (col.type() == ColumnType.STRING) {
-      String value = ((StringColumn) col).get(r);
-      try {
-        return attr.valueOf(value);
-      } catch (ParseException e) {
-        throw new IllegalArgumentException("Error converting " + value + " to nominal", e);
+  public DataFrame toDataFrame(StructType schema) {
+    List<Tuple> rows = new ArrayList<>();
+    int colCount = table.columnCount();
+    for (int rowIndex = 0; rowIndex < table.rowCount(); rowIndex++) {
+      Object[] row = new Object[colCount];
+      for (int colIndex = 0; colIndex < colCount; colIndex++) {
+        Column<?> col = table.column(colIndex);
+        if (!col.isMissing(rowIndex)) {
+          row[colIndex] =
+              col.type().equals(ColumnType.INSTANT)
+                  ? LocalDateTime.ofInstant(((InstantColumn) col).get(rowIndex), ZoneOffset.UTC)
+                  : col.get(rowIndex);
+        }
       }
+      rows.add(Tuple.of(row, schema));
     }
-    if (col instanceof NumericColumn) {
-      return ((NumericColumn<?>) col).getDouble(r);
-    }
-    throw new IllegalStateException(
-        "Error converting " + col.type() + " column " + col.name() + " to Smile");
-  }
-
-  private Attribute colAsAttribute(Column<?> col) {
-    return col.type() == ColumnType.STRING
-        ? colAsNominalAttribute(col)
-        : new NumericAttribute(col.name());
-  }
 
-  private NominalAttribute colAsNominalAttribute(Column<?> col) {
-    Column<?> unique = col.unique();
-    return new NominalAttribute(
-        col.name(),
-        unique
-            .mapInto(o -> o.toString(), StringColumn.create(col.name(), unique.size()))
-            .asObjectArray());
+    return DataFrame.of(rows, schema.boxed(rows));
   }
 
-  private static enum AttributeType {
-    NUMERIC,
-    NOMINAL
+  private DataType toSmileType(ColumnType type) {
+    if (type.equals(ColumnType.BOOLEAN)) {
+      return DataTypes.BooleanType;
+    } else if (type.equals(ColumnType.DOUBLE)) {
+      return DataTypes.DoubleType;
+    } else if (type.equals(ColumnType.FLOAT)) {
+      return DataTypes.FloatType;
+    } else if (type.equals(ColumnType.INSTANT)) {
+      return DataTypes.DateTimeType;
+    } else if (type.equals(ColumnType.INTEGER)) {
+      return DataTypes.IntegerType;
+    } else if (type.equals(ColumnType.LOCAL_DATE)) {
+      return DataTypes.DateType;
+    } else if (type.equals(ColumnType.LOCAL_DATE_TIME)) {
+      return DataTypes.DateTimeType;
+    } else if (type.equals(ColumnType.LOCAL_TIME)) {
+      return DataTypes.TimeType;
+    } else if (type.equals(ColumnType.LONG)) {
+      return DataTypes.LongType;
+    } else if (type.equals(ColumnType.SHORT)) {
+      return DataTypes.ShortType;
+    } else if (type.equals(ColumnType.STRING)) {
+      return DataTypes.StringType;
+    } else if (type.equals(ColumnType.TEXT)) {
+      return DataTypes.StringType;
+    }
+    throw new IllegalStateException("Unsupported column type " + type);
   }
 }
diff --git a/docs-src/dist/userguide/ml/Moneyball Linear regression.md b/docs-src/dist/userguide/ml/Moneyball Linear regression.md
index afa19d70..7440b15d 100644
--- a/docs-src/dist/userguide/ml/Moneyball Linear regression.md
+++ b/docs-src/dist/userguide/ml/Moneyball Linear regression.md
@@ -18,7 +18,7 @@ To do regression modeling in Tablesaw, we'll first need to import Smile:
 <dependency>
   <groupId>com.github.haifengl</groupId>
   <artifactId>smile-core</artifactId>
-  <version>1.5.3</version>
+  <version>2.0.0</version>
 </dependency>
 ```
 
@@ -107,7 +107,7 @@ Our plot shows a strong linear relation between the two.
 Let's create our first predictive model using linear regression, with runDifference as the sole explanatory variable. Here we use Smile's OLS (Ordinary Least Squares) regression model.
 
 ```Java
-OLS winsModel = new OLS(moneyball.select("W", "RD").smile().numericDataset("RD"));
+LinearModel winsModel = OLS.fit(Formula.lhs("RD"), moneyball.select("W", "RD").smile().toDataFrame());
 ```
 
 If we print our "winsModel", it produces the output below:
@@ -152,7 +152,7 @@ We'd expect 95 wins when we outscore opponents by 135 runs.
 It's time to go deeper again and see how we can model Runs Scored and Runs Allowed. The approach the A's took was to model Runs Scored using team On-base percent (OBP) and team Slugging Average (SLG). In Tablesaw, we write:
 
 ```java
-OLS runsScored = new OLS(moneyball.smile().numericDataset("RS", "OBP", "SLG"));
+LinearModel runsScored = OLS.fit(Formula.lhs("RS"), moneyball.select("RS", "OBP", "SLG").smile().toDataFrame());
 ```
 
 
@@ -188,7 +188,7 @@ It looks great.  It's also important to plot the predicted (or "fitted") value
 Our Scatter class can create this plot directly from the model:
 
 ```java
-double[] fitted = runsScored2.fitted();
+double[] fitted = runsScored2.fittedValues();
 double[] resids = runsScored2.residuals();
 
 ScatterPlot.show("Runs Scored from OBP and SLG", "Fitted", fitted, "Residuals", resids);
@@ -207,7 +207,7 @@ SLG &amp; OBP -&gt; Runs Scored -&gt; Run Difference -&gt; Regular Season Wins
 Of course, we haven't modeled the Runs Allowed side of Run Difference. We could use pitching and field stats to do this, but the A's cleverly used the same two variables (SLG and OBP), but now looked at how their opponent's performed against the A's. We could do the same as these data are encoded in the dataset as OOBP and OSLG.
 
 ```java
-OLS runsAllowed = new OLS(moneyball.smile().numericDataset("RA", "OOBP", "OSLG"));
+LinearModel runsAllowed = OLS.fit(Formula.lhs("RA"), moneyball.select("RA", "OOBP", "OSLG").smile().toDataFrame());
 
 > Linear Model:
 
@@ -233,7 +233,7 @@ This model also looks good, but you'd want to look at the plots again, and do ot
 Finally, we can tie this all together and see how well wins is predicted when we consider both offensive and defensive stats. 
 
 ```java
-OLS winsFinal = new OLS(moneyball.smile().numericDataset("W", "OOBP", "OBP", "OSLG", "SLG"));
+LinearModel winsFinal = new OLS(Formula.lhs("W"), moneyball.select("W", "OOBP", "OBP", "OSLG", "SLG").smile().toDataFrame());
 ```
 
 The output isn't shown, but we get an R squared of .89. Again this is quite good. 


Test patch:
diff --git a/core/src/test/java/tech/tablesaw/conversion/smile/SmileConverterTest.java b/core/src/test/java/tech/tablesaw/conversion/smile/SmileConverterTest.java
index d43d7c5f..80d59dd1 100644
--- a/core/src/test/java/tech/tablesaw/conversion/smile/SmileConverterTest.java
+++ b/core/src/test/java/tech/tablesaw/conversion/smile/SmileConverterTest.java
@@ -14,17 +14,33 @@
 
 package tech.tablesaw.conversion.smile;
 
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 import java.io.IOException;
-import java.util.Arrays;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
 import org.junit.jupiter.api.Test;
-import smile.classification.RandomForest;
-import smile.data.AttributeDataset;
+import smile.data.DataFrame;
+import smile.data.formula.Formula;
+import smile.regression.LinearModel;
 import smile.regression.OLS;
+import tech.tablesaw.api.BooleanColumn;
+import tech.tablesaw.api.DateColumn;
+import tech.tablesaw.api.DateTimeColumn;
+import tech.tablesaw.api.DoubleColumn;
+import tech.tablesaw.api.FloatColumn;
+import tech.tablesaw.api.InstantColumn;
+import tech.tablesaw.api.IntColumn;
+import tech.tablesaw.api.LongColumn;
+import tech.tablesaw.api.ShortColumn;
+import tech.tablesaw.api.StringColumn;
 import tech.tablesaw.api.Table;
+import tech.tablesaw.api.TextColumn;
+import tech.tablesaw.api.TimeColumn;
 
 public class SmileConverterTest {
 
@@ -33,50 +49,43 @@ public class SmileConverterTest {
     Table moneyball = Table.read().csv("../data/baseball.csv");
     moneyball.addColumns(
         moneyball.numberColumn("RS").subtract(moneyball.numberColumn("RA")).setName("RD"));
-    OLS winsModel = new OLS(moneyball.select("W", "RD").smile().numericDataset("RD"));
-    assertNotNull(winsModel.toString());
-  }
 
-  @Test
-  public void regressionWithStratifiedSampleTest() throws IOException {
-    Table moneyball = Table.read().csv("../data/baseball.csv");
-    Table[] splits = moneyball.stratifiedSampleSplit(moneyball.stringColumn("Team"), 0.6);
-    Table stratifiedMoneyBall = splits[0];
-    stratifiedMoneyBall.addColumns(
-        stratifiedMoneyBall
-            .numberColumn("RS")
-            .subtract(stratifiedMoneyBall.numberColumn("RA"))
-            .setName("RD"));
-    OLS winsModel = new OLS(stratifiedMoneyBall.select("W", "RD").smile().numericDataset("RD"));
+    LinearModel winsModel =
+        OLS.fit(Formula.lhs("RD"), moneyball.select("W", "RD").smile().toDataFrame());
     assertNotNull(winsModel.toString());
   }
 
   @Test
-  public void classification() throws IOException {
-    Table moneyball = Table.read().csv("../data/baseball.csv");
-    RandomForest playoffsModel =
-        new RandomForest(moneyball.smile().nominalDataset("Playoffs", "RS", "RA", "OBP"), 1);
-    assertNotNull(playoffsModel.toString());
-  }
-
-  @Test
-  public void nominalDatasetToString() throws IOException {
-    Table moneyball = Table.read().csv("../data/baseball.csv");
-    AttributeDataset dataset =
-        moneyball.smile().nominalDataset("Playoffs", "League", "RS", "RA", "OBP");
-    assertNotNull(dataset.toString());
-  }
-
-  @Test
-  public void columnNames() throws IOException {
-    Table moneyball = Table.read().csv("../data/baseball.csv");
-    String responseColumnName = "Playoffs";
-    String[] variableColumnNames = new String[] {"League", "RS", "RA", "OBP"};
-    AttributeDataset dataset =
-        moneyball.smile().nominalDataset(responseColumnName, variableColumnNames);
-    String[] resultNames =
-        Arrays.stream(dataset.attributes()).map(attr -> attr.getName()).toArray(String[]::new);
-    assertEquals(responseColumnName, dataset.responseAttribute().getName());
-    assertArrayEquals(variableColumnNames, resultNames);
+  public void allColumnTypes() throws IOException {
+    Table table = Table.create();
+    table.addColumns(BooleanColumn.create("boolean", new boolean[] {true, false}));
+    table.addColumns(DoubleColumn.create("double", new double[] {1.2, 3.4}));
+    table.addColumns(FloatColumn.create("float", new float[] {5.6f, 7.8f}));
+    table.addColumns(
+        InstantColumn.create(
+            "instant",
+            new Instant[] {
+              Instant.ofEpochMilli(1578452479123l), Instant.ofEpochMilli(1578451111111l)
+            }));
+    table.addColumns(IntColumn.create("int", new int[] {8, 9}));
+    table.addColumns(
+        DateColumn.create(
+            "date", new LocalDate[] {LocalDate.of(2020, 01, 01), LocalDate.of(2020, 01, 07)}));
+    table.addColumns(
+        DateTimeColumn.create(
+            "datetime",
+            new LocalDateTime[] {
+              LocalDateTime.ofInstant(Instant.ofEpochMilli(1333352479123l), ZoneOffset.UTC),
+              LocalDateTime.ofInstant(Instant.ofEpochMilli(1333333333333l), ZoneOffset.UTC)
+            }));
+    table.addColumns(
+        TimeColumn.create(
+            "time", new LocalTime[] {LocalTime.of(8, 37, 48), LocalTime.of(8, 59, 06)}));
+    table.addColumns(LongColumn.create("long", new long[] {3l, 4l}));
+    table.addColumns(ShortColumn.create("short", new short[] {1, 2}));
+    table.addColumns(StringColumn.create("string", new String[] {"james", "bond"}));
+    table.addColumns(TextColumn.create("text", new String[] {"foo", "bar"}));
+    DataFrame dataframe = table.smile().toDataFrame();
+    assertEquals(2, dataframe.nrows());
   }
 }