Repository: nens/threedi-modelchecker

Issue:
Add additional checks
The following fields of v2_manhole should not be null:
- bottom level
- surface level
- drain level

**Some extra checks:**
- manhole.bottom_level > manhole.surface_level
- manhole.bottom_level > manhole.drain_level
- global_settings.max_timestep >= global_settings.timestep
- global_settings.timestep >= global_settings.min_timestep

warning: manhole.surface_level < manhole.drain_level

**Cross-table reference checks (might be more difficult):**
- pumpstation.lower_stop_level > manhole.bottom_level
- pipe.invert_level >= manhole.bottom_level

warning: weir.crest_level > manhole.bottom_level manhole





Comments:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index a893a3e..8890c52 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -5,7 +5,7 @@ Changelog of threedi-modelchecker
 0.9 (unreleased)
 ----------------
 
-- Nothing changed yet.
+- Fixed some misconfigured checks, see https://github.com/nens/threedi-modelchecker/issues/10.
 
 
 0.8 (2019-11-26)
diff --git a/threedi_modelchecker/config.py b/threedi_modelchecker/config.py
index e98d55e..d51cce5 100644
--- a/threedi_modelchecker/config.py
+++ b/threedi_modelchecker/config.py
@@ -173,14 +173,6 @@ RANGE_CHECKS = [
         column=models.Weir.friction_value,
         criterion_valid=models.Weir.friction_value >= 0,
     ),
-    GeneralCheck(
-        column=models.Manhole.bottom_level,
-        criterion_valid=models.Manhole.bottom_level >= models.Manhole.surface_level,
-    ),
-    GeneralCheck(
-        column=models.Manhole.bottom_level,
-        criterion_valid=models.Manhole.bottom_level >= models.Manhole.drain_level,
-    ),
     GeneralCheck(
         column=models.GlobalSetting.maximum_sim_time_step,
         criterion_valid=models.GlobalSetting.maximum_sim_time_step >= models.GlobalSetting.sim_time_step,  # noqa: E501
@@ -210,6 +202,7 @@ OTHER_CHECKS = [
     Use0DFlowCheck()
 ]
 
+
 CONDITIONAL_CHECKS = [
     ConditionalCheck(
         criterion=(models.ConnectionNode.id == models.Manhole.connection_node_id),
@@ -226,12 +219,6 @@ CONDITIONAL_CHECKS = [
                              < models.CrossSectionLocation.bank_level)
         )
     ),
-    ConditionalCheck(
-        criterion=(models.GlobalSetting.timestep_plus == True),
-        check=NotNullCheck(
-            column=models.GlobalSetting.maximum_sim_time_step,
-        )
-    ),
     ConditionalCheck(
         criterion=or_(
             models.GlobalSetting.initial_groundwater_level_file != None,
@@ -374,7 +361,7 @@ CONDITIONAL_CHECKS = [
         ).filter(
             models.Pipe.invert_level_end_point < models.Manhole.bottom_level,
         ),
-        message="Pipe.invert_level_end_point should be higher or equal than "
+        message="Pipe.invert_level_end_point should be higher than or equal to "
                 "Manhole.bottom_level"
     ),
     QueryCheck(
@@ -387,9 +374,36 @@ CONDITIONAL_CHECKS = [
         ).filter(
             models.Pipe.invert_level_start_point < models.Manhole.bottom_level,  # noqa: E501
         ),
-        message="Pipe.invert_level_start_point should be higher or equal than "
+        message="Pipe.invert_level_start_point should be higher than or equal to "
                 "Manhole.bottom_level"
-    )
+    ),
+    QueryCheck(
+        column=models.Manhole.bottom_level,
+        invalid=Query(models.Manhole).filter(
+            models.Manhole.drain_level < models.Manhole.bottom_level,
+            models.Manhole.calculation_type == constants.CalculationTypeNode.CONNECTED
+        ),
+        message="Manhole.drain_level >= Manhole.bottom_level when "
+                "Manhole.calculation_type is CONNECTED"
+    ),
+    QueryCheck(
+        column=models.Manhole.drain_level,
+        invalid=Query(models.Manhole).filter(
+            models.Manhole.calculation_type == constants.CalculationTypeNode.CONNECTED,
+            models.Manhole.drain_level == None
+        ),
+        message="Manhole.drain_level cannot be null when Manhole.calculation_type is "
+                "CONNECTED"
+    ),
+    QueryCheck(
+        column=models.GlobalSetting.maximum_sim_time_step,
+        invalid=Query(models.GlobalSetting).filter(
+            models.GlobalSetting.timestep_plus == True,
+            models.GlobalSetting.maximum_sim_time_step == None
+        ),
+        message="GlobalSettings.maximum_sim_time_step cannot be null when "
+                "GlobalSettings.timestep_plus is True."
+    ),
 ]
 
 
diff --git a/threedi_modelchecker/threedi_model/models.py b/threedi_modelchecker/threedi_model/models.py
index 284b1d4..342c9de 100644
--- a/threedi_modelchecker/threedi_model/models.py
+++ b/threedi_modelchecker/threedi_model/models.py
@@ -325,9 +325,9 @@ class Manhole(Base):
     shape = Column(String(4))
     width = Column(Float)
     length = Column(Float)
-    surface_level = Column(Float, nullable=False)
+    surface_level = Column(Float)
     bottom_level = Column(Float, nullable=False)
-    drain_level = Column(Float, nullable=False)
+    drain_level = Column(Float)
     sediment_level = Column(Float)
     manhole_indicator = Column(Integer)
     calculation_type = Column(IntegerEnum(constants.CalculationTypeNode))


Test patch:
diff --git a/tests/test_checks_base.py b/tests/test_checks_base.py
index c9a8814..11cfae2 100644
--- a/tests/test_checks_base.py
+++ b/tests/test_checks_base.py
@@ -599,3 +599,56 @@ def test_general_check_modulo_operator(session):
     invalid = modulo_check.get_invalid(session)
     assert len(invalid) == 1
     assert invalid[0].id == global_settings_remainder.id
+
+
+def test_query_check_manhole_drain_level_calc_type_2(session):
+    # manhole.drain_level can be null, but if manhole.calculation_type == 2 (Connected)
+    # then manhole.drain_level >= manhole.bottom_level
+    factories.ManholeFactory(drain_level=None)
+    factories.ManholeFactory(drain_level=1)
+    m3_error = factories.ManholeFactory(
+        drain_level=None,
+        calculation_type=constants.CalculationTypeNode.CONNECTED
+    )  # drain_level cannot be null when calculation_type is CONNECTED
+    m4_error = factories.ManholeFactory(
+        drain_level=1,
+        bottom_level=2,
+        calculation_type=constants.CalculationTypeNode.CONNECTED
+    )  # bottom_level  >= drain_level when calculation_type is CONNECTED
+    factories.ManholeFactory(
+        drain_level=1,
+        bottom_level=0,
+        calculation_type=constants.CalculationTypeNode.CONNECTED
+    )
+    factories.ManholeFactory(
+        drain_level=None,
+        bottom_level=0,
+        calculation_type=constants.CalculationTypeNode.EMBEDDED
+    )
+
+    query_drn_lvl_st_bttm_lvl = Query(models.Manhole).filter(
+        models.Manhole.drain_level < models.Manhole.bottom_level,
+        models.Manhole.calculation_type == constants.CalculationTypeNode.CONNECTED
+    )
+    query_invalid_not_null = Query(models.Manhole).filter(
+        models.Manhole.calculation_type == constants.CalculationTypeNode.CONNECTED,
+        models.Manhole.drain_level == None
+    )
+    check_drn_lvl_gt_bttm_lvl = QueryCheck(
+        column=models.Manhole.bottom_level,
+        invalid=query_drn_lvl_st_bttm_lvl,
+        message="Manhole.drain_level >= Manhoole.bottom_level when "
+                "Manhole.calculation_type is CONNECTED"
+    )
+    check_invalid_not_null = QueryCheck(
+        column=models.Manhole.drain_level,
+        invalid=query_invalid_not_null,
+        message="Manhole.drain_level cannot be null when Manhole.calculation_type is "
+                "CONNECTED"
+    )
+    errors1 = check_drn_lvl_gt_bttm_lvl.get_invalid(session)
+    errors2 = check_invalid_not_null.get_invalid(session)
+    assert len(errors1) == 1
+    assert len(errors2) == 1
+    assert m3_error.id == errors2[0].id
+    assert m4_error.id == errors1[0].id
diff --git a/tests/test_checks_factories.py b/tests/test_checks_factories.py
index 828076f..4086a31 100644
--- a/tests/test_checks_factories.py
+++ b/tests/test_checks_factories.py
@@ -22,7 +22,7 @@ def test_gen_not_unique_checks():
 
 def test_gen_not_null_checks():
     not_null_checks = generate_not_null_checks(models.Manhole.__table__)
-    assert len(not_null_checks) == 7
+    assert len(not_null_checks) == 5
     not_null_check_columns = [check.column for check in not_null_checks]
     assert models.Manhole.id in not_null_check_columns
     assert models.Manhole.code in not_null_check_columns