Repository: deeplearning4j/deeplearning4j
Issue: Transfer learning: 0 dropout doesn't remove/override existing layer dropout

```
            netToTest = new TransferLearning.Builder(net)
                    .fineTuneConfiguration(new FineTuneConfiguration.Builder()
                            .dropOut(0.0)
                    .build()).build();
```

The original dropout is still present on the existing layers. Reason: .dropOut(0.0) does dropOut((IDropOut)null) hence it's treated as equivalent to "not set"


Changed files:
deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java
deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java
deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java

diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java
index da2b116696..893b8d68de 100644
--- a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java
+++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java
@@ -2,11 +2,14 @@ package org.deeplearning4j.nn.transferlearning;
 
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
 import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
+import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
 import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
+import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint;
 import org.deeplearning4j.nn.conf.distribution.NormalDistribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.*;
 import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer;
+import org.deeplearning4j.nn.conf.weightnoise.DropConnect;
 import org.deeplearning4j.nn.graph.ComputationGraph;
 import org.deeplearning4j.nn.weights.WeightInit;
 import org.junit.Test;
@@ -23,6 +26,7 @@ import java.util.Collections;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 /**
  * Created by susaneraly on 2/17/17.
@@ -421,4 +425,41 @@ public class TransferLearningCompGraphTest {
 //        assertEquals(confExpected, graph.getConfiguration());
         assertEquals(confExpected.toJson(), graph.getConfiguration().toJson());
     }
+
+
+    @Test
+    public void testObjectOverrides(){
+        //https://github.com/deeplearning4j/deeplearning4j/issues/4368
+        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
+                .dropOut(0.5)
+                .weightNoise(new DropConnect(0.5))
+                .l2(0.5)
+                .constrainWeights(new UnitNormConstraint())
+                .graphBuilder()
+                .addInputs("in")
+                .addLayer("layer", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in")
+                .setOutputs("layer")
+                .build();
+
+        ComputationGraph orig = new ComputationGraph(conf);
+        orig.init();
+
+        FineTuneConfiguration ftc = new FineTuneConfiguration.Builder()
+                .dropOut(0)
+                .weightNoise(null)
+                .constraints(null)
+                .l2(0.0)
+                .build();
+
+        ComputationGraph transfer = new TransferLearning.GraphBuilder(orig)
+                .fineTuneConfiguration(ftc)
+                .build();
+
+        DenseLayer l = (DenseLayer) transfer.getLayer(0).conf().getLayer();
+
+        assertNull(l.getIDropout());
+        assertNull(l.getWeightNoise());
+        assertNull(l.getConstraints());
+        assertEquals(0.0, l.getL2(), 0.0);
+    }
 }
diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java
index 443eb1e7e5..86f2599489 100644
--- a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java
+++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java
@@ -6,12 +6,14 @@ import org.deeplearning4j.nn.conf.BackpropType;
 import org.deeplearning4j.nn.conf.GradientNormalization;
 import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
 import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
+import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint;
 import org.deeplearning4j.nn.conf.distribution.NormalDistribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.*;
 import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor;
 import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor;
 import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor;
+import org.deeplearning4j.nn.conf.weightnoise.DropConnect;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
 import org.deeplearning4j.nn.weights.WeightInit;
 import org.junit.Test;
@@ -578,5 +580,38 @@ public class TransferLearningMLNTest {
         assertEquals(expectedParams, modelNow.params());
     }
 
+    @Test
+    public void testObjectOverrides(){
+        //https://github.com/deeplearning4j/deeplearning4j/issues/4368
+        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+                .dropOut(0.5)
+                .weightNoise(new DropConnect(0.5))
+                .l2(0.5)
+                .constrainWeights(new UnitNormConstraint())
+                .list()
+                .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
+                .build();
+
+        MultiLayerNetwork orig = new MultiLayerNetwork(conf);
+        orig.init();
+
+        FineTuneConfiguration ftc = new FineTuneConfiguration.Builder()
+                .dropOut(0)
+                .weightNoise(null)
+                .constraints(null)
+                .l2(0.0)
+                .build();
+
+        MultiLayerNetwork transfer = new TransferLearning.Builder(orig)
+                .fineTuneConfiguration(ftc)
+                .build();
+
+        DenseLayer l = (DenseLayer) transfer.getLayer(0).conf().getLayer();
+
+        assertNull(l.getIDropout());
+        assertNull(l.getWeightNoise());
+        assertNull(l.getConstraints());
+        assertEquals(0.0, l.getL2(), 0.0);
+    }
 
 }
diff --git a/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java b/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java
index c68b7430bd..d2de16ab1b 100644
--- a/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java
+++ b/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java
@@ -1,7 +1,6 @@
 package org.deeplearning4j.nn.transferlearning;
 
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
@@ -17,6 +16,7 @@ import org.deeplearning4j.nn.weights.WeightInit;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.activations.IActivation;
 import org.nd4j.linalg.learning.config.IUpdater;
+import org.nd4j.linalg.primitives.Optional;
 import org.nd4j.shade.jackson.annotation.JsonInclude;
 import org.nd4j.shade.jackson.annotation.JsonTypeInfo;
 import org.nd4j.shade.jackson.core.JsonProcessingException;
@@ -25,14 +25,16 @@ import java.io.IOException;
 import java.util.List;
 
 /**
- * Created by Alex on 21/02/2017.
+ * Configuration for fine tuning. Note that values set here will override values for all non-frozen layers
+ *
+ * @author Alex Black
+ * @author Susan Eraly
  */
 @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "type")
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @NoArgsConstructor
 @AllArgsConstructor
 @Data
-@Builder(builderClassName = "Builder")
 public class FineTuneConfiguration {
 
     protected IActivation activationFn;
@@ -43,9 +45,9 @@ public class FineTuneConfiguration {
     protected Double l2;
     protected Double l1Bias;
     protected Double l2Bias;
-    protected IDropout dropout;
-    protected IWeightNoise weightNoise;
-    protected IUpdater iUpdater;
+    protected Optional<IDropout> dropout;
+    protected Optional<IWeightNoise> weightNoise;
+    protected IUpdater updater;
     protected IUpdater biasUpdater;
     protected Boolean miniBatch;
     protected Integer maxNumLineSearchIterations;
@@ -53,12 +55,10 @@ public class FineTuneConfiguration {
     protected OptimizationAlgorithm optimizationAlgo;
     protected StepFunction stepFunction;
     protected Boolean minimize;
-    protected GradientNormalization gradientNormalization;
+    protected Optional<GradientNormalization> gradientNormalization;
     protected Double gradientNormalizationThreshold;
     protected ConvolutionMode convolutionMode;
-    protected List<LayerConstraint> constraints;
-    protected Boolean hasBiasConstraints;
-    protected Boolean hasWeightConstraints;
+    protected Optional<List<LayerConstraint>> constraints;
 
     protected Boolean pretrain;
     protected Boolean backprop;
@@ -69,20 +69,103 @@ public class FineTuneConfiguration {
     protected WorkspaceMode trainingWorkspaceMode;
     protected WorkspaceMode inferenceWorkspaceMode;
 
-    //Lombok builder. Note that the code below ADDS OR OVERRIDES the lombok implementation; the final builder class
-    // is the composite of the lombok parts and the parts defined here
-    //partial implementation to allow public no-arg constructor (lombok default is package private)
-    //Plus some implementations to match NeuralNetConfiguration builder methods
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /*
+     * Can't use Lombok @Builder annotation due to optionals (otherwise we have a bunch of ugly .x(Optional<T> value)
+      * methods - lombok builder doesn't support excluding fields? :(
+     * Note the use of optional here: gives us 3 states...
+     * 1. Null: not set
+     * 2. Optional (empty): set to null
+     * 3. Optional (not empty): set to specific value
+     *
+     * Obviously, having null only makes sense for some things (dropout, etc) whereas null for other things doesn't
+     * make sense
+     */
     public static class Builder {
-        public Builder() {}
+        private IActivation activation;
+        private WeightInit weightInit;
+        private Double biasInit;
+        private Distribution dist;
+        private Double l1;
+        private Double l2;
+        private Double l1Bias;
+        private Double l2Bias;
+        private Optional<IDropout> dropout;
+        private Optional<IWeightNoise> weightNoise;
+        private IUpdater updater;
+        private IUpdater biasUpdater;
+        private Boolean miniBatch;
+        private Integer maxNumLineSearchIterations;
+        private Long seed;
+        private OptimizationAlgorithm optimizationAlgo;
+        private StepFunction stepFunction;
+        private Boolean minimize;
+        private Optional<GradientNormalization> gradientNormalization;
+        private Double gradientNormalizationThreshold;
+        private ConvolutionMode convolutionMode;
+        private Optional<List<LayerConstraint>> constraints;
+        private Boolean pretrain;
+        private Boolean backprop;
+        private BackpropType backpropType;
+        private Integer tbpttFwdLength;
+        private Integer tbpttBackLength;
+        private WorkspaceMode trainingWorkspaceMode;
+        private WorkspaceMode inferenceWorkspaceMode;
 
-        public Builder seed(int seed) {
-            this.seed = (long) seed;
+        public Builder() {
+
+        }
+
+        public Builder activation(IActivation activationFn) {
+            this.activation = activationFn;
             return this;
         }
 
-        public Builder seed(long seed) {
-            this.seed = seed;
+        public Builder activation(Activation activation) {
+            this.activation = activation.getActivationFunction();
+            return this;
+        }
+
+        public Builder weightInit(WeightInit weightInit) {
+            this.weightInit = weightInit;
+            return this;
+        }
+
+        public Builder biasInit(double biasInit) {
+            this.biasInit = biasInit;
+            return this;
+        }
+
+        public Builder dist(Distribution dist) {
+            this.dist = dist;
+            return this;
+        }
+
+        public Builder l1(double l1) {
+            this.l1 = l1;
+            return this;
+        }
+
+        public Builder l2(double l2) {
+            this.l2 = l2;
+            return this;
+        }
+
+        public Builder l1Bias(double l1Bias) {
+            this.l1Bias = l1Bias;
+            return this;
+        }
+
+        public Builder l2Bias(double l2Bias) {
+            this.l2Bias = l2Bias;
+            return this;
+        }
+
+        public Builder dropout(IDropout dropout) {
+            this.dropout = Optional.ofNullable(dropout);
             return this;
         }
 
@@ -93,19 +176,122 @@ public class FineTuneConfiguration {
             return dropout(new Dropout(dropout));
         }
 
-        public Builder activation(Activation activation) {
-            this.activationFn = activation.getActivationFunction();
+        public Builder weightNoise(IWeightNoise weightNoise) {
+            this.weightNoise = Optional.ofNullable(weightNoise);
             return this;
         }
 
         public Builder updater(IUpdater updater) {
-            return iUpdater(updater);
+            this.updater = updater;
+            return this;
         }
 
         @Deprecated
         public Builder updater(Updater updater) {
             return updater(updater.getIUpdaterWithDefaultConfig());
         }
+
+        public Builder biasUpdater(IUpdater biasUpdater) {
+            this.biasUpdater = biasUpdater;
+            return this;
+        }
+
+        public Builder miniBatch(boolean miniBatch) {
+            this.miniBatch = miniBatch;
+            return this;
+        }
+
+        public Builder maxNumLineSearchIterations(int maxNumLineSearchIterations) {
+            this.maxNumLineSearchIterations = maxNumLineSearchIterations;
+            return this;
+        }
+
+        public Builder seed(long seed) {
+            this.seed = seed;
+            return this;
+        }
+
+        public Builder seed(int seed){
+            return seed((long)seed);
+        }
+
+        public Builder optimizationAlgo(OptimizationAlgorithm optimizationAlgo) {
+            this.optimizationAlgo = optimizationAlgo;
+            return this;
+        }
+
+        public Builder stepFunction(StepFunction stepFunction) {
+            this.stepFunction = stepFunction;
+            return this;
+        }
+
+        public Builder minimize(boolean minimize) {
+            this.minimize = minimize;
+            return this;
+        }
+
+        public Builder gradientNormalization(GradientNormalization gradientNormalization) {
+            this.gradientNormalization = Optional.ofNullable(gradientNormalization);
+            return this;
+        }
+
+        public Builder gradientNormalizationThreshold(double gradientNormalizationThreshold) {
+            this.gradientNormalizationThreshold = gradientNormalizationThreshold;
+            return this;
+        }
+
+        public Builder convolutionMode(ConvolutionMode convolutionMode) {
+            this.convolutionMode = convolutionMode;
+            return this;
+        }
+
+        public Builder constraints(List<LayerConstraint> constraints) {
+            this.constraints = Optional.ofNullable(constraints);
+            return this;
+        }
+
+        public Builder pretrain(boolean pretrain) {
+            this.pretrain = pretrain;
+            return this;
+        }
+
+        public Builder backprop(boolean backprop) {
+            this.backprop = backprop;
+            return this;
+        }
+
+        public Builder backpropType(BackpropType backpropType) {
+            this.backpropType = backpropType;
+            return this;
+        }
+
+        public Builder tbpttFwdLength(int tbpttFwdLength) {
+            this.tbpttFwdLength = tbpttFwdLength;
+            return this;
+        }
+
+        public Builder tbpttBackLength(int tbpttBackLength) {
+            this.tbpttBackLength = tbpttBackLength;
+            return this;
+        }
+
+        public Builder trainingWorkspaceMode(WorkspaceMode trainingWorkspaceMode) {
+            this.trainingWorkspaceMode = trainingWorkspaceMode;
+            return this;
+        }
+
+        public Builder inferenceWorkspaceMode(WorkspaceMode inferenceWorkspaceMode) {
+            this.inferenceWorkspaceMode = inferenceWorkspaceMode;
+            return this;
+        }
+
+        public FineTuneConfiguration build() {
+            return new FineTuneConfiguration(activation, weightInit, biasInit, dist, l1, l2, l1Bias, l2Bias, dropout, weightNoise, updater, biasUpdater, miniBatch, maxNumLineSearchIterations, seed, optimizationAlgo, stepFunction, minimize, gradientNormalization, gradientNormalizationThreshold, convolutionMode, constraints, pretrain, backprop, backpropType, tbpttFwdLength, tbpttBackLength, trainingWorkspaceMode, inferenceWorkspaceMode);
+        }
+
+        public String toString() {
+            return "FineTuneConfiguration.Builder(activation=" + this.activation + ", weightInit=" + this.weightInit + ", biasInit=" + this.biasInit + ", dist=" + this.dist + ", l1=" + this.l1 + ", l2=" + this.l2 + ", l1Bias=" + this.l1Bias + ", l2Bias=" + this.l2Bias + ", dropout=" + this.dropout + ", weightNoise=" + this.weightNoise + ", updater=" + this.updater + ", biasUpdater=" + this.biasUpdater + ", miniBatch=" + this.miniBatch + ", maxNumLineSearchIterations=" + this.maxNumLineSearchIterations + ", seed=" + this.seed + ", optimizationAlgo=" + this.optimizationAlgo + ", stepFunction=" + this.stepFunction + ", minimize=" + this.minimize + ", gradientNormalization=" + this.gradientNormalization + ", gradientNormalizationThreshold=" + this.gradientNormalizationThreshold + ", convolutionMode=" + this.convolutionMode + ", constraints=" + this.constraints + ", pretrain=" + this.pretrain + ", backprop=" + this.backprop + ", backpropType=" + this.backpropType + ", tbpttFwdLength=" + this.tbpttFwdLength + ", tbpttBackLength=" + this.tbpttBackLength + ", trainingWorkspaceMode=" + this.trainingWorkspaceMode + ", inferenceWorkspaceMode=" + this.inferenceWorkspaceMode + ")";
+        }
     }
 
 
@@ -123,7 +309,9 @@ public class FineTuneConfiguration {
 
         if (l != null) {
             if (dropout != null)
-                l.setIDropout(dropout);
+                l.setIDropout(dropout.orElse(null));
+            if(constraints != null)
+                l.setConstraints(constraints.orElse(null));
         }
 
         if (l != null && l instanceof BaseLayer) {
@@ -146,17 +334,17 @@ public class FineTuneConfiguration {
             if (l2Bias != null)
                 bl.setL2Bias(l2Bias);
             if (gradientNormalization != null)
-                bl.setGradientNormalization(gradientNormalization);
+                bl.setGradientNormalization(gradientNormalization.orElse(null));
             if (gradientNormalizationThreshold != null)
                 bl.setGradientNormalizationThreshold(gradientNormalizationThreshold);
-            if (iUpdater != null){
-                bl.setIUpdater(iUpdater);
+            if (updater != null){
+                bl.setIUpdater(updater);
             }
             if (biasUpdater != null){
                 bl.setBiasUpdater(biasUpdater);
             }
             if (weightNoise != null){
-                bl.setWeightNoise(weightNoise);
+                bl.setWeightNoise(weightNoise.orElse(null));
             }
         }
         if (miniBatch != null)
@@ -185,9 +373,10 @@ public class FineTuneConfiguration {
             ((BaseLayer) l).setDist(null);
         }
 
-        //Perform validation. This also sets the defaults for updaters. For example, Updater.RMSProp -> set rmsDecay
+        //Perform validation
         if (l != null) {
-            LayerValidation.generalValidation(l.getLayerName(), l, dropout, l2, l2Bias, l1, l1Bias, dist, constraints, null, null);
+            LayerValidation.generalValidation(l.getLayerName(), l, get(dropout), l2, l2Bias, l1, l1Bias,
+                    dist, get(constraints), null, null);
         }
 
         //Also: update the LR, L1 and L2 maps, based on current config (which might be different to original config)
@@ -198,6 +387,13 @@ public class FineTuneConfiguration {
         }
     }
 
+    private static <T> T get(Optional<T> optional){
+        if(optional == null){
+            return null;
+        }
+        return optional.orElse(null);
+    }
+
     public void applyToMultiLayerConfiguration(MultiLayerConfiguration conf) {
         if (pretrain != null)
             conf.setPretrain(pretrain);
@@ -243,9 +439,9 @@ public class FineTuneConfiguration {
         if (l2Bias != null)
             confBuilder.setL2Bias(l2Bias);
         if (dropout != null)
-            confBuilder.setIdropOut(dropout);
-        if (iUpdater != null)
-            confBuilder.updater(iUpdater);
+            confBuilder.setIdropOut(dropout.orElse(null));
+        if (updater != null)
+            confBuilder.updater(updater);
         if(biasUpdater != null)
             confBuilder.biasUpdater(biasUpdater);
         if (miniBatch != null)
@@ -261,7 +457,7 @@ public class FineTuneConfiguration {
         if (minimize != null)
             confBuilder.setMinimize(minimize);
         if (gradientNormalization != null)
-            confBuilder.setGradientNormalization(gradientNormalization);
+            confBuilder.setGradientNormalization(gradientNormalization.orElse(null));
         if (gradientNormalizationThreshold != null)
             confBuilder.setGradientNormalizationThreshold(gradientNormalizationThreshold);
         if (trainingWorkspaceMode != null)