Repository: telstra/open-kilda
Issue: flowDeleteV2 is leaving excess meter on Centec switch

**Steps to reproduce:**

1. create a flow on a Centec switch;
2. delete the flow;

**Actual result:**
excess meter on a Centec switch after deleting the flow

**Expected result:**
system shouldn't leave excess meter

Changed files:
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowInstallCommand.java
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressRemoveFlowModFactory.java
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowRemoveCommand.java
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressFlowModFactory.java
src-java/floodlight-service/floodlight-modules/src/test/java/org/openkilda/floodlight/command/flow/ingress/IngressCommandInstallTest.java
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentRemoveCommand.java
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentInstallCommand.java
src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressInstallFlowModFactory.java

diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentInstallCommand.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentInstallCommand.java
index 01c2f9873..d9ec2d829 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentInstallCommand.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentInstallCommand.java
@@ -64,6 +64,10 @@ public class IngressFlowSegmentInstallCommand extends IngressFlowSegmentCommand
         if (encapsulation.getType() == FlowEncapsulationType.VXLAN) {
             required.add(SwitchFeature.NOVIFLOW_COPY_FIELD);
         }
+        if (metadata.isMultiTable()) {
+            required.add(SwitchFeature.MULTI_TABLE);
+        }
+
         return required;
     }
 
diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentRemoveCommand.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentRemoveCommand.java
index 6b08beffd..8c9c5e5d2 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentRemoveCommand.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/IngressFlowSegmentRemoveCommand.java
@@ -26,6 +26,7 @@ import org.openkilda.model.FlowEndpoint;
 import org.openkilda.model.FlowTransitEncapsulation;
 import org.openkilda.model.MeterConfig;
 import org.openkilda.model.MeterId;
+import org.openkilda.model.SwitchFeature;
 import org.openkilda.model.SwitchId;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -70,7 +71,7 @@ public class IngressFlowSegmentRemoveCommand extends IngressFlowSegmentCommand {
     @Override
     protected List<OFFlowMod> makeIngressModMessages(MeterId effectiveMeterId) {
         List<OFFlowMod> ofMessages = super.makeIngressModMessages(effectiveMeterId);
-        if (rulesContext != null) {
+        if (getSwitchFeatures().contains(SwitchFeature.MULTI_TABLE) && rulesContext != null) {
             if (rulesContext.isRemoveCustomerCatchRule()) {
                 ofMessages.add(getFlowModFactory().makeCustomerPortSharedCatchMessage());
             }
diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowInstallCommand.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowInstallCommand.java
index b75325dec..22b8f9322 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowInstallCommand.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowInstallCommand.java
@@ -25,12 +25,14 @@ import org.openkilda.messaging.MessageContext;
 import org.openkilda.model.FlowEndpoint;
 import org.openkilda.model.MeterConfig;
 import org.openkilda.model.MeterId;
+import org.openkilda.model.SwitchFeature;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.projectfloodlight.openflow.protocol.OFFlowMod;
 
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 
@@ -79,6 +81,16 @@ public class OneSwitchFlowInstallCommand extends OneSwitchFlowCommand {
         return ofMessages;
     }
 
+    @Override
+    protected Set<SwitchFeature> getRequiredFeatures() {
+        Set<SwitchFeature> required = super.getRequiredFeatures();
+        if (metadata.isMultiTable()) {
+            required.add(SwitchFeature.MULTI_TABLE);
+        }
+
+        return required;
+    }
+
     @Override
     protected SegmentAction getSegmentAction() {
         return SegmentAction.INSTALL;
diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowRemoveCommand.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowRemoveCommand.java
index 63509bbc5..796d6204f 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowRemoveCommand.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/OneSwitchFlowRemoveCommand.java
@@ -25,6 +25,7 @@ import org.openkilda.messaging.MessageContext;
 import org.openkilda.model.FlowEndpoint;
 import org.openkilda.model.MeterConfig;
 import org.openkilda.model.MeterId;
+import org.openkilda.model.SwitchFeature;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -65,7 +66,7 @@ public class OneSwitchFlowRemoveCommand extends OneSwitchFlowCommand {
     @Override
     protected List<OFFlowMod> makeIngressModMessages(MeterId effectiveMeterId) {
         List<OFFlowMod> ofMessages = super.makeIngressModMessages(effectiveMeterId);
-        if (rulesContext != null) {
+        if (getSwitchFeatures().contains(SwitchFeature.MULTI_TABLE) && rulesContext != null) {
             if (rulesContext.isRemoveCustomerCatchRule()) {
                 ofMessages.add(getFlowModFactory().makeCustomerPortSharedCatchMessage());
             }
diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressFlowModFactory.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressFlowModFactory.java
index 1746cf321..622f21e01 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressFlowModFactory.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressFlowModFactory.java
@@ -26,7 +26,6 @@ import org.openkilda.model.SwitchFeature;
 import org.openkilda.model.cookie.Cookie;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import lombok.AccessLevel;
 import lombok.Getter;
@@ -35,7 +34,6 @@ import org.projectfloodlight.openflow.protocol.OFFactory;
 import org.projectfloodlight.openflow.protocol.OFFlowMod;
 import org.projectfloodlight.openflow.protocol.OFFlowModFlags;
 import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
-import org.projectfloodlight.openflow.protocol.instruction.OFInstructionWriteMetadata;
 import org.projectfloodlight.openflow.protocol.match.MatchField;
 import org.projectfloodlight.openflow.types.EthType;
 import org.projectfloodlight.openflow.types.OFPort;
@@ -109,8 +107,7 @@ public abstract class IngressFlowModFactory {
                 .setMatch(of.buildMatch()
                                   .setExact(MatchField.IN_PORT, OFPort.of(endpoint.getPortNumber()))
                                   .build())
-                .setInstructions(ImmutableList.of(
-                        of.instructions().gotoTable(TableId.of(SwitchManager.PRE_INGRESS_TABLE_ID))))
+                .setInstructions(makeCustomerPortSharedCatchInstructions())
                 .build();
     }
 
@@ -121,12 +118,6 @@ public abstract class IngressFlowModFactory {
      */
     public OFFlowMod makeLldpInputCustomerFlowMessage() {
         FlowEndpoint endpoint = command.getEndpoint();
-        RoutingMetadata metadata = RoutingMetadata.builder().lldpFlag(true).build(switchFeatures);
-        OFInstructionWriteMetadata writeMetadata = of.instructions().buildWriteMetadata()
-                .setMetadata(metadata.getValue())
-                .setMetadataMask(metadata.getMask())
-                .build();
-
         return flowModBuilderFactory.makeBuilder(of, SwitchManager.INPUT_TABLE_ID)
                 .setPriority(SwitchManager.LLDP_INPUT_CUSTOMER_PRIORITY)
                 .setCookie(U64.of(Cookie.encodeLldpInputCustomer(endpoint.getPortNumber())))
@@ -135,9 +126,8 @@ public abstract class IngressFlowModFactory {
                         .setExact(MatchField.IN_PORT, OFPort.of(endpoint.getPortNumber()))
                         .setExact(MatchField.ETH_TYPE, EthType.LLDP)
                         .build())
-                .setInstructions(ImmutableList.of(
-                        of.instructions().gotoTable(TableId.of(SwitchManager.PRE_INGRESS_TABLE_ID)),
-                        writeMetadata))
+                .setInstructions(makeConnectedDevicesMatchInstructions(
+                        RoutingMetadata.builder().lldpFlag(true).build(switchFeatures)))
                 .build();
     }
 
@@ -148,12 +138,6 @@ public abstract class IngressFlowModFactory {
      */
     public OFFlowMod makeArpInputCustomerFlowMessage() {
         FlowEndpoint endpoint = command.getEndpoint();
-        RoutingMetadata metadata = RoutingMetadata.builder().arpFlag(true).build(switchFeatures);
-        OFInstructionWriteMetadata writeMetadata = of.instructions().buildWriteMetadata()
-                .setMetadata(metadata.getValue())
-                .setMetadataMask(metadata.getMask())
-                .build();
-
         return flowModBuilderFactory.makeBuilder(of, SwitchManager.INPUT_TABLE_ID)
                 .setPriority(SwitchManager.ARP_INPUT_CUSTOMER_PRIORITY)
                 .setCookie(U64.of(Cookie.encodeArpInputCustomer(endpoint.getPortNumber())))
@@ -162,9 +146,8 @@ public abstract class IngressFlowModFactory {
                         .setExact(MatchField.IN_PORT, OFPort.of(endpoint.getPortNumber()))
                         .setExact(MatchField.ETH_TYPE, EthType.ARP)
                         .build())
-                .setInstructions(ImmutableList.of(
-                        of.instructions().gotoTable(TableId.of(SwitchManager.PRE_INGRESS_TABLE_ID)),
-                        writeMetadata))
+                .setInstructions(makeConnectedDevicesMatchInstructions(
+                        RoutingMetadata.builder().arpFlag(true).build(switchFeatures)))
                 .build();
     }
 
@@ -177,4 +160,8 @@ public abstract class IngressFlowModFactory {
     }
 
     protected abstract List<OFInstruction> makeForwardMessageInstructions(OFFactory of, MeterId effectiveMeterId);
+
+    protected abstract List<OFInstruction> makeCustomerPortSharedCatchInstructions();
+
+    protected abstract List<OFInstruction> makeConnectedDevicesMatchInstructions(RoutingMetadata metadata);
 }
diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressInstallFlowModFactory.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressInstallFlowModFactory.java
index 14469d0bc..364ad82ed 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressInstallFlowModFactory.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressInstallFlowModFactory.java
@@ -19,9 +19,11 @@ import org.openkilda.floodlight.command.flow.ingress.IngressFlowSegmentBase;
 import org.openkilda.floodlight.switchmanager.SwitchManager;
 import org.openkilda.floodlight.utils.OfAdapter;
 import org.openkilda.floodlight.utils.OfFlowModBuilderFactory;
+import org.openkilda.floodlight.utils.metadata.RoutingMetadata;
 import org.openkilda.model.MeterId;
 import org.openkilda.model.SwitchFeature;
 
+import com.google.common.collect.ImmutableList;
 import net.floodlightcontroller.core.IOFSwitch;
 import org.projectfloodlight.openflow.protocol.OFFactory;
 import org.projectfloodlight.openflow.protocol.action.OFAction;
@@ -60,6 +62,22 @@ public abstract class IngressInstallFlowModFactory extends IngressFlowModFactory
         return instructions;
     }
 
+    @Override
+    protected List<OFInstruction> makeCustomerPortSharedCatchInstructions() {
+        return ImmutableList.of(
+                of.instructions().gotoTable(TableId.of(SwitchManager.PRE_INGRESS_TABLE_ID)));
+    }
+
+    @Override
+    protected List<OFInstruction> makeConnectedDevicesMatchInstructions(RoutingMetadata metadata) {
+        return ImmutableList.of(
+                of.instructions().gotoTable(TableId.of(SwitchManager.PRE_INGRESS_TABLE_ID)),
+                of.instructions().buildWriteMetadata()
+                        .setMetadata(metadata.getValue())
+                        .setMetadataMask(metadata.getMask())
+                        .build());
+    }
+
     protected abstract List<OFAction> makeTransformActions();
 
     protected abstract List<OFInstruction> makeMetadataInstructions();
diff --git a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressRemoveFlowModFactory.java b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressRemoveFlowModFactory.java
index aca3f6cbf..201b59309 100644
--- a/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressRemoveFlowModFactory.java
+++ b/src-java/floodlight-service/floodlight-modules/src/main/java/org/openkilda/floodlight/command/flow/ingress/of/IngressRemoveFlowModFactory.java
@@ -17,6 +17,7 @@ package org.openkilda.floodlight.command.flow.ingress.of;
 
 import org.openkilda.floodlight.command.flow.ingress.IngressFlowSegmentBase;
 import org.openkilda.floodlight.utils.OfFlowModBuilderFactory;
+import org.openkilda.floodlight.utils.metadata.RoutingMetadata;
 import org.openkilda.model.MeterId;
 import org.openkilda.model.SwitchFeature;
 
@@ -39,4 +40,14 @@ public abstract class IngressRemoveFlowModFactory extends IngressFlowModFactory
     protected List<OFInstruction> makeForwardMessageInstructions(OFFactory of, MeterId effectiveMeterId) {
         return Collections.emptyList();
     }
+
+    @Override
+    protected List<OFInstruction> makeCustomerPortSharedCatchInstructions() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    protected List<OFInstruction> makeConnectedDevicesMatchInstructions(RoutingMetadata metadata) {
+        return Collections.emptyList();
+    }
 }
diff --git a/src-java/floodlight-service/floodlight-modules/src/test/java/org/openkilda/floodlight/command/flow/ingress/IngressCommandInstallTest.java b/src-java/floodlight-service/floodlight-modules/src/test/java/org/openkilda/floodlight/command/flow/ingress/IngressCommandInstallTest.java
index 79af6f74e..d5351a824 100644
--- a/src-java/floodlight-service/floodlight-modules/src/test/java/org/openkilda/floodlight/command/flow/ingress/IngressCommandInstallTest.java
+++ b/src-java/floodlight-service/floodlight-modules/src/test/java/org/openkilda/floodlight/command/flow/ingress/IngressCommandInstallTest.java
@@ -21,7 +21,9 @@ import org.openkilda.floodlight.command.flow.FlowSegmentReport;
 import org.openkilda.floodlight.error.SwitchErrorResponseException;
 import org.openkilda.floodlight.error.SwitchOperationException;
 import org.openkilda.floodlight.error.UnsupportedSwitchOperationException;
+import org.openkilda.model.SwitchFeature;
 
+import net.floodlightcontroller.core.IOFSwitch;
 import org.junit.Assert;
 import org.junit.Test;
 import org.projectfloodlight.openflow.protocol.OFBadRequestCode;
@@ -34,9 +36,16 @@ import org.projectfloodlight.openflow.protocol.instruction.OFInstructionMeter;
 import org.projectfloodlight.openflow.protocol.instruction.OFInstructionWriteActions;
 
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 abstract class IngressCommandInstallTest extends IngressCommandTest {
+    @Override
+    protected void switchFeaturesSetup(IOFSwitch target, Set<SwitchFeature> features) {
+        features.add(SwitchFeature.MULTI_TABLE);
+        super.switchFeaturesSetup(target, features);
+    }
+
     @Test
     public void noMeterRequested() throws Exception {
         IngressFlowSegmentBase command = makeCommand(endpointIngressOneVlan, null, makeMetadata());