Repository: deepjavalibrary/djl
Pull request #2075 (2022-10-11 16:29:55)

Fix randomColor
## Description ##

Brief description of what this PR is about

Fix randomColor because Random.nextInt() upper bound is exclusive, upper bound has to be 256 to have values from 0 to 255.




Patch:
diff --git a/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java b/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java
index 7017370dd..c208b6ae4 100644
--- a/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java
+++ b/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java
@@ -327,7 +327,7 @@ public class BitmapImageFactory extends ImageFactory {
 
         private int randomColor() {
             return Color.rgb(
-                    RandomUtils.nextInt(255), RandomUtils.nextInt(255), RandomUtils.nextInt(255));
+                    RandomUtils.nextInt(256), RandomUtils.nextInt(256), RandomUtils.nextInt(256));
         }
 
         private int darker(int color) {
diff --git a/basicdataset/src/main/java/ai/djl/basicdataset/tabular/AmesRandomAccess.java b/basicdataset/src/main/java/ai/djl/basicdataset/tabular/AmesRandomAccess.java
index b9b4e8edb..e0e8ee7cf 100644
--- a/basicdataset/src/main/java/ai/djl/basicdataset/tabular/AmesRandomAccess.java
+++ b/basicdataset/src/main/java/ai/djl/basicdataset/tabular/AmesRandomAccess.java
@@ -29,7 +29,6 @@ import java.io.Reader;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -44,6 +43,9 @@ import java.util.Set;
  * <p>Test Set: 1459 Records
  *
  * <p>Can enable/disable features Set one hot vector for categorical variables
+ *
+ * <p>Call {@link Builder#addAllFeatures()} to include all features from the dataset. The label is a
+ * numeric column named "saleprice".
  */
 public class AmesRandomAccess extends CsvDataset {
 
@@ -200,15 +202,29 @@ public class AmesRandomAccess extends CsvDataset {
         public Builder addFeature(String name, boolean onehotEncode) {
             parseFeatures();
             if (af.categorical.contains(name)) {
-                Map<String, Integer> map = af.featureToMap.get(name);
-                if (map == null) {
-                    return addCategoricalFeature(name, onehotEncode);
-                }
-                return addCategoricalFeature(name, map, onehotEncode);
+                return addCategoricalFeature(name, onehotEncode);
             }
             return addNumericFeature(name);
         }
 
+        /**
+         * Adds all features to the features set.
+         *
+         * @return this builder
+         */
+        public Builder addAllFeatures() {
+            if (features.isEmpty()) {
+                parseFeatures();
+                for (String name : af.featureArray) {
+                    addFeature(name);
+                }
+            }
+            if (labels.isEmpty()) {
+                addNumericLabel("saleprice");
+            }
+            return this;
+        }
+
         /**
          * Returns the available features of this dataset.
          *
@@ -226,15 +242,6 @@ public class AmesRandomAccess extends CsvDataset {
          */
         @Override
         public AmesRandomAccess build() {
-            if (features.isEmpty()) {
-                parseFeatures();
-                for (String name : af.featureArray) {
-                    addFeature(name);
-                }
-            }
-            if (labels.isEmpty()) {
-                addNumericLabel("saleprice");
-            }
             return new AmesRandomAccess(this);
         }
 
@@ -260,6 +267,5 @@ public class AmesRandomAccess extends CsvDataset {
 
         List<String> featureArray;
         Set<String> categorical;
-        Map<String, Map<String, Integer>> featureToMap;
     }
 }
diff --git a/basicdataset/src/main/java/ai/djl/basicdataset/tabular/CsvDataset.java b/basicdataset/src/main/java/ai/djl/basicdataset/tabular/CsvDataset.java
index 5de3eccc9..702ceaa5f 100644
--- a/basicdataset/src/main/java/ai/djl/basicdataset/tabular/CsvDataset.java
+++ b/basicdataset/src/main/java/ai/djl/basicdataset/tabular/CsvDataset.java
@@ -155,7 +155,6 @@ public class CsvDataset extends TabularDataset {
          * @return the new {@link CsvDataset}
          */
         public CsvDataset build() {
-            validate();
             return new CsvDataset(this);
         }
     }
diff --git a/basicdataset/src/main/java/ai/djl/basicdataset/tabular/TabularDataset.java b/basicdataset/src/main/java/ai/djl/basicdataset/tabular/TabularDataset.java
index 9a8380c1c..9a0febfe0 100644
--- a/basicdataset/src/main/java/ai/djl/basicdataset/tabular/TabularDataset.java
+++ b/basicdataset/src/main/java/ai/djl/basicdataset/tabular/TabularDataset.java
@@ -44,6 +44,13 @@ public abstract class TabularDataset extends RandomAccessDataset {
         super(builder);
         features = builder.features;
         labels = builder.labels;
+
+        if (features.isEmpty()) {
+            throw new IllegalArgumentException("Missing features.");
+        }
+        if (labels.isEmpty() && !builder.allowNoLabels) {
+            throw new IllegalArgumentException("Missing labels.");
+        }
     }
 
     /**
@@ -135,6 +142,7 @@ public abstract class TabularDataset extends RandomAccessDataset {
 
         protected List<Feature> features;
         protected List<Feature> labels;
+        protected boolean allowNoLabels;
 
         protected BaseBuilder() {
             features = new ArrayList<>();
@@ -283,17 +291,13 @@ public abstract class TabularDataset extends RandomAccessDataset {
         }
 
         /**
-         * Validates the builder to ensure it is correct.
+         * Indicates the dataset should not have any labels.
          *
-         * @throws IllegalArgumentException if there is an error with the builder arguments
+         * @return this builder
          */
-        protected void validate() {
-            if (features.isEmpty()) {
-                throw new IllegalArgumentException("Missing features.");
-            }
-            if (labels.isEmpty()) {
-                throw new IllegalArgumentException("Missing labels.");
-            }
+        public T noLabels() {
+            allowNoLabels = true;
+            return self();
         }
     }
 }
diff --git a/basicdataset/src/main/resources/ai/djl/basicdataset/tabular/ames.json b/basicdataset/src/main/resources/ai/djl/basicdataset/tabular/ames.json
index 07ac15318..336365f10 100644
--- a/basicdataset/src/main/resources/ai/djl/basicdataset/tabular/ames.json
+++ b/basicdataset/src/main/resources/ai/djl/basicdataset/tabular/ames.json
@@ -30,7 +30,6 @@
     "exterqual",
     "extercond",
     "foundation",
-    "bsmtqual",
     "bsmtcond",
     "bsmtexposure",
     "bsmtfintype1",
@@ -103,6 +102,7 @@
     "exterqual",
     "extercond",
     "foundation",
+    "bsmtqual",
     "bsmtcond",
     "bsmtexposure",
     "bsmtfintype1",
@@ -124,354 +124,5 @@
     "miscfeature",
     "saletype",
     "salecondition"
-  ],
-  "featureToMap": {
-    "lotconfig": {
-      "CulDSac": 3,
-      "Corner": 2,
-      "FR3": 4,
-      "FR2": 1,
-      "Inside": 0
-    },
-    "garagefinish": {
-      "Fin": 2,
-      "NA": 3,
-      "RFn": 0,
-      "Unf": 1
-    },
-    "poolqc": {
-      "NA": 0,
-      "Ex": 1,
-      "Fa": 2,
-      "Gd": 3
-    },
-    "kitchenqual": {
-      "Ex": 2,
-      "Fa": 3,
-      "Gd": 0,
-      "TA": 1
-    },
-    "functional": {
-      "Mod": 4,
-      "Min2": 3,
-      "Sev": 6,
-      "Maj2": 5,
-      "Typ": 0,
-      "Min1": 1,
-      "Maj1": 2
-    },
-    "mszoning": {
-      "FV": 3,
-      "RH": 4,
-      "RL": 0,
-      "RM": 1,
-      "C (all)": 2
-    },
-    "bsmtexposure": {
-      "No": 0,
-      "Mn": 2,
-      "NA": 4,
-      "Av": 3,
-      "Gd": 1
-    },
-    "heatingqc": {
-      "Ex": 0,
-      "Fa": 3,
-      "Gd": 1,
-      "TA": 2,
-      "Po": 4
-    },
-    "foundation": {
-      "CBlock": 1,
-      "BrkTil": 2,
-      "Wood": 3,
-      "PConc": 0,
-      "Slab": 4,
-      "Stone": 5
-    },
-    "utilities": {
-      "NoSeWa": 1,
-      "AllPub": 0
-    },
-    "paveddrive": {
-      "P": 2,
-      "Y": 0,
-      "N": 1
-    },
-    "exterior1st": {
-      "AsbShng": 8,
-      "Stucco": 9,
-      "BrkComm": 10,
-      "AsphShn": 11,
-      "CBlock": 14,
-      "WdShing": 5,
-      "Stone": 12,
-      "BrkFace": 4,
-      "MetalSd": 1,
-      "Wd Sdng": 2,
-      "HdBoard": 3,
-      "VinylSd": 0,
-      "Plywood": 7,
-      "CemntBd": 6,
-      "ImStucc": 13
-    },
-    "roofstyle": {
-      "Mansard": 3,
-      "Shed": 5,
-      "Gambrel": 2,
-      "Gable": 0,
-      "Flat": 4,
-      "Hip": 1
-    },
-    "street": {
-      "Pave": 0,
-      "Grvl": 1
-    },
-    "alley": {
-      "NA": 0,
-      "Grvl": 1,
-      "Pave": 2
-    },
-    "heating": {
-      "Grav": 2,
-      "OthW": 4,
-      "Floor": 5,
-      "Wall": 3,
-      "GasA": 0,
-      "GasW": 1
-    },
-    "bldgtype": {
-      "Twnhs": 4,
-      "1Fam": 0,
-      "TwnhsE": 3,
-      "2fmCon": 1,
-      "Duplex": 2
-    },
-    "masvnrtype": {
-      "BrkFace": 0,
-      "NA": 4,
-      "None": 1,
-      "Stone": 2,
-      "BrkCmn": 3
-    },
-    "extercond": {
-      "Ex": 4,
-      "Fa": 2,
-      "TA": 0,
-      "Gd": 1,
-      "Po": 3
-    },
-    "garagetype": {
-      "CarPort": 3,
-      "Detchd": 1,
-      "NA": 4,
-      "Basment": 5,
-      "Attchd": 0,
-      "2Types": 6,
-      "BuiltIn": 2
-    },
-    "electrical": {
-      "FuseA": 2,
-      "NA": 5,
-      "FuseF": 1,
-      "SBrkr": 0,
-      "Mix": 4,
-      "FuseP": 3
-    },
-    "exterqual": {
-      "Ex": 2,
-      "Fa": 3,
-      "Gd": 0,
-      "TA": 1
-    },
-    "landslope": {
-      "Mod": 1,
-      "Sev": 2,
-      "Gtl": 0
-    },
-    "bsmtcond": {
-      "NA": 2,
-      "Fa": 3,
-      "TA": 0,
-      "Gd": 1,
-      "Po": 4
-    },
-    "landcontour": {
-      "Lvl": 0,
-      "Low": 2,
-      "Bnk": 1,
-      "HLS": 3
-    },
-    "fireplacequ": {
-      "NA": 0,
-      "Ex": 4,
-      "Fa": 3,
-      "TA": 1,
-      "Gd": 2,
-      "Po": 5
-    },
-    "housestyle": {
-      "1Story": 1,
-      "2.5Fin": 7,
-      "2.5Unf": 6,
-      "2Story": 0,
-      "SLvl": 5,
-      "1.5Unf": 3,
-      "SFoyer": 4,
-      "1.5Fin": 2
-    },
-    "exterior2nd": {
-      "Stucco": 8,
-      "AsbShng": 9,
-      "AsphShn": 12,
-      "CBlock": 15,
-      "CmentBd": 6,
-      "Brk Cmn": 10,
-      "Wd Shng": 2,
-      "Stone": 13,
-      "BrkFace": 7,
-      "MetalSd": 1,
-      "HdBoard": 3,
-      "Wd Sdng": 5,
-      "VinylSd": 0,
-      "Plywood": 4,
-      "ImStucc": 11,
-      "Other": 14
-    },
-    "condition1": {
-      "RRAe": 4,
-      "RRNn": 5,
-      "RRAn": 6,
-      "PosA": 7,
-      "Artery": 3,
-      "RRNe": 8,
-      "Norm": 0,
-      "PosN": 2,
-      "Feedr": 1
-    },
-    "garagequal": {
-      "NA": 3,
-      "Ex": 4,
-      "Fa": 1,
-      "TA": 0,
-      "Gd": 2,
-      "Po": 5
-    },
-    "roofmatl": {
-      "Tar\u0026Grv": 5,
-      "ClyTile": 7,
-      "WdShngl": 1,
-      "Membran": 4,
-      "Metal": 2,
-      "Roll": 6,
-      "CompShg": 0,
-      "WdShake": 3
-    },
-    "condition2": {
-      "RRAe": 7,
-      "RRNn": 2,
-      "Norm": 0,
-      "PosA": 5,
-      "RRAn": 6,
-      "PosN": 4,
-      "Feedr": 3,
-      "Artery": 1
-    },
-    "saletype": {
-      "New": 1,
-      "Oth": 8,
-      "Con": 7,
-      "ConLw": 6,
-      "WD": 0,
-      "CWD": 5,
-      "COD": 2,
-      "ConLI": 4,
-      "ConLD": 3
-    },
-    "garagecond": {
-      "NA": 2,
-      "Ex": 5,
-      "Fa": 1,
-      "TA": 0,
-      "Gd": 3,
-      "Po": 4
-    },
-    "centralair": {
-      "Y": 0,
-      "N": 1
-    },
-    "lotshape": {
-      "IR2": 2,
-      "IR1": 1,
-      "IR3": 3,
-      "Reg": 0
-    },
-    "neighborhood": {
-      "NWAmes": 6,
-      "BrDale": 22,
-      "Somerst": 5,
-      "Timber": 16,
-      "Gilbert": 17,
-      "SawyerW": 12,
-      "NPkVill": 20,
-      "Sawyer": 9,
-      "NoRidge": 3,
-      "NridgHt": 10,
-      "StoneBr": 18,
-      "Veenker": 1,
-      "BrkSide": 8,
-      "NAmes": 11,
-      "CollgCr": 0,
-      "MeadowV": 14,
-      "Blmngtn": 21,
-      "Blueste": 24,
-      "IDOTRR": 13,
-      "Edwards": 15,
-      "ClearCr": 19,
-      "Mitchel": 4,
-      "OldTown": 7,
-      "Crawfor": 2,
-      "SWISU": 23
-    },
-    "fence": {
-      "GdPrv": 3,
-      "NA": 0,
-      "MnPrv": 1,
-      "GdWo": 2,
-      "MnWw": 4
-    },
-    "bsmtfintype2": {
-      "Rec": 4,
-      "NA": 2,
-      "BLQ": 1,
-      "ALQ": 3,
-      "LwQ": 5,
-      "Unf": 0,
-      "GLQ": 6
-    },
-    "miscfeature": {
-      "TenC": 4,
-      "NA": 0,
-      "Shed": 1,
-      "Othr": 3,
-      "Gar2": 2
-    },
-    "salecondition": {
-      "AdjLand": 3,
-      "Alloca": 4,
-      "Partial": 2,
-      "Family": 5,
-      "Normal": 0,
-      "Abnorml": 1
-    },
-    "bsmtfintype1": {
-      "Rec": 3,
-      "NA": 5,
-      "ALQ": 1,
-      "BLQ": 4,
-      "LwQ": 6,
-      "Unf": 2,
-      "GLQ": 0
-    }
-  }
+  ]
 }
diff --git a/extensions/tablesaw/src/main/java/ai/djl/tablesaw/TablesawDataset.java b/extensions/tablesaw/src/main/java/ai/djl/tablesaw/TablesawDataset.java
index aaa277255..183775154 100644
--- a/extensions/tablesaw/src/main/java/ai/djl/tablesaw/TablesawDataset.java
+++ b/extensions/tablesaw/src/main/java/ai/djl/tablesaw/TablesawDataset.java
@@ -104,7 +104,6 @@ public class TablesawDataset extends TabularDataset {
          * @return the new {@link TablesawDataset}
          */
         public TablesawDataset build() {
-            validate();
             return new TablesawDataset(this);
         }
     }


Test patch:
diff --git a/basicdataset/src/test/java/ai/djl/basicdataset/AmesRandomAccessTest.java b/basicdataset/src/test/java/ai/djl/basicdataset/AmesRandomAccessTest.java
index d0a008aeb..ab58b8cb8 100644
--- a/basicdataset/src/test/java/ai/djl/basicdataset/AmesRandomAccessTest.java
+++ b/basicdataset/src/test/java/ai/djl/basicdataset/AmesRandomAccessTest.java
@@ -64,6 +64,7 @@ public class AmesRandomAccessTest {
                             .addFeature("MiscVal")
                             .addFeature("id")
                             .addFeature("alley", true)
+                            .addNumericLabel("saleprice")
                             .setSampling(32, true)
                             .build();
 
@@ -75,7 +76,7 @@ public class AmesRandomAccessTest {
             Record record = amesRandomAccess.get(manager, 0);
             NDList data = record.getData();
             NDList labels = record.getLabels();
-            Assert.assertEquals(data.head().toFloatArray(), new float[] {0, 1, 1, 0, 0});
+            Assert.assertEquals(data.head().toFloatArray(), new float[] {0, 1, 0, 1, 0});
             Assert.assertEquals(labels.singletonOrThrow().toFloatArray()[0], 208500.0);
 
             try (Trainer trainer = model.newTrainer(config)) {
@@ -86,4 +87,39 @@ public class AmesRandomAccessTest {
             }
         }
     }
+
+    @Test
+    public void testAmesRandomAccessAllFeatures() throws IOException, TranslateException {
+        try (NDManager manager = NDManager.newBaseManager()) {
+            AmesRandomAccess amesRandomAccess =
+                    AmesRandomAccess.builder()
+                            .optUsage(Dataset.Usage.TRAIN)
+                            .addAllFeatures()
+                            .setSampling(32, true)
+                            .build();
+
+            amesRandomAccess.prepare();
+
+            long size = amesRandomAccess.size();
+            Assert.assertEquals(size, 1460);
+
+            Record record = amesRandomAccess.get(manager, 0);
+            NDList data = record.getData();
+            Assert.assertEquals(data.head().size(), 79);
+        }
+    }
+
+    @Test
+    public void testAmesNoFeatures() {
+        Assert.assertThrows(
+                () -> {
+                    AmesRandomAccess amesRandomAccess =
+                            AmesRandomAccess.builder()
+                                    .optUsage(Dataset.Usage.TRAIN)
+                                    .setSampling(32, true)
+                                    .build();
+
+                    amesRandomAccess.prepare();
+                });
+    }
 }
diff --git a/basicdataset/src/test/java/ai/djl/basicdataset/DailyDelhiClimateTest.java b/basicdataset/src/test/java/ai/djl/basicdataset/DailyDelhiClimateTest.java
index a7efa6496..6a68c6431 100644
--- a/basicdataset/src/test/java/ai/djl/basicdataset/DailyDelhiClimateTest.java
+++ b/basicdataset/src/test/java/ai/djl/basicdataset/DailyDelhiClimateTest.java
@@ -51,6 +51,7 @@ public class DailyDelhiClimateTest {
                             .optUsage(Dataset.Usage.TEST)
                             .addFeature("humidity")
                             .addFeature("wind_speed")
+                            .noLabels()
                             .setSampling(32, true)
                             .build();