Repository: signaux-faibles/predictsignauxfaibles

Issue:
Make SIREN and SIRET string not int
### Problem
The dataframe retrieved by `SFDataset` contains SIRET and SIREN that are typed as `int64` which means that their leading 0 is ignored. So we have

```python
# this fails
dataset.data.siren == "012345678"

# this works
dataset.data.siren == 12345678
```

which is not right

### Solution
Figure out a way to force the `siret` and `siren` columns to be read as strings.

### ⚠️ 
I haven't checked yet but there's a chance that this typing issue not due to pandas but to MongoDB.

Comments:

Patch:
diff --git a/predictsignauxfaibles/data.py b/predictsignauxfaibles/data.py
index 1fb9e9c..70835a0 100644
--- a/predictsignauxfaibles/data.py
+++ b/predictsignauxfaibles/data.py
@@ -151,6 +151,12 @@ class SFDataset:
                 for feat in missing:
                     self.data[feat] = NAN
 
+        # force SIREN and SIRET to be strings
+        if "siren" in self.data.columns:
+            self.data.siren = self.data.siren.astype(str)
+        if "siret" in self.data.columns:
+            self.data.siret = self.data.siret.astype(str)
+
         return self
 
     def raise_if_empty(self):
diff --git a/predictsignauxfaibles/pipelines.py b/predictsignauxfaibles/pipelines.py
index 7f2df8b..3e564d8 100644
--- a/predictsignauxfaibles/pipelines.py
+++ b/predictsignauxfaibles/pipelines.py
@@ -11,6 +11,11 @@ from predictsignauxfaibles.preprocessors import (
     acoss_make_avg_delta_dette_par_effectif,
 )
 
+from predictsignauxfaibles.redressements import (
+    Redressement,
+    redressement_urssaf_covid,
+)
+
 
 class MissingDataError(Exception):
     """
@@ -20,29 +25,30 @@ class MissingDataError(Exception):
 
 def run_pipeline(data: pd.DataFrame, pipeline: List[namedtuple]):
     """
-    Run a pipeline of Preprocessor objects on a dataframe
+    Run a pipeline of Preprocessor or Redressement objects (aka "steps") on a dataframe
     Args:
-        pipeline: a list of Preprocessor objects (see predictsignauxfaibles.preprocessors)
+        pipeline: a list of Preprocessor or Redressement objects
+          (see predictsignauxfaibles.preprocessors or predictsignauxfaibles.redressements)
     """
     logging.info("Checking that input columns are all there.")
-    for preprocessor in pipeline:
-        if not set(preprocessor.input).issubset(data.columns):
-            missing_cols = set(preprocessor.input) - set(data.columns)
+    for step in pipeline:
+        if not set(step.input).issubset(data.columns):
+            missing_cols = set(step.input) - set(data.columns)
             error_message = (
-                f"Missing variables {missing_cols} in order to run {preprocessor.name}."
+                f"Missing variables {missing_cols} in order to run {step.name}."
             )
             raise MissingDataError(error_message)
 
     logging.info("Running pipeline on data.")
     data = data.copy()
-    for i, preprocessor in enumerate(pipeline):
-        logging.info(f"STEP {i+1}: {preprocessor.name}")
-        data = preprocessor.function(data)
-        if preprocessor.output is None:
+    for i, step in enumerate(pipeline):
+        logging.info(f"STEP {i+1}: {step.name}")
+        data = step.function(data)
+        if step.output is None:
             continue
-        if not set(preprocessor.output).issubset(data.columns):
-            missing_output_cols = set(preprocessor.output) - set(data.columns)
-            warning_message = f"STEP {i+1}: function {preprocessor.function.__name__} \
+        if not set(step.output).issubset(data.columns):
+            missing_output_cols = set(step.output) - set(data.columns)
+            warning_message = f"STEP {i+1}: function {step.function.__name__} \
 did not produce expected output {missing_output_cols}"
             logging.warning(warning_message)
             continue
@@ -52,6 +58,8 @@ did not produce expected output {missing_output_cols}"
 
 # Pipelines
 
+# Preprocessors
+
 DEFAULT_PIPELINE = [
     Preprocessor(
         "Remove Administrations",
@@ -106,5 +114,23 @@ SMALL_PIPELINE = [
     ),
 ]
 
+# Redressements
+
+REDRESSEMENTS_PIPELINE = [
+    Redressement(
+        "Redressement URSSAF evolution dette Juillet 2020",
+        redressement_urssaf_covid,
+        input=[
+            "montant_part_ouvriere_latest",
+            "montant_part_patronale_latest",
+            "montant_part_ouvriere_july2020",
+            "montant_part_patronale_july2020",
+            "cotisation_moy12m_latest",
+            "group_final",
+        ],
+        output=["group_final_regle_urssaf"],
+    ),
+]
+
 # This is useful for automatic testing
-ALL_PIPELINES = [DEFAULT_PIPELINE, SMALL_PIPELINE]
+ALL_PIPELINES = [DEFAULT_PIPELINE, SMALL_PIPELINE, REDRESSEMENTS_PIPELINE]
diff --git a/predictsignauxfaibles/redressements.py b/predictsignauxfaibles/redressements.py
new file mode 100644
index 0000000..58dffa5
--- /dev/null
+++ b/predictsignauxfaibles/redressements.py
@@ -0,0 +1,48 @@
+from collections import namedtuple
+
+import pandas as pd
+
+Redressement = namedtuple("Redressement", ["name", "function", "input", "output"])
+
+
+def redressement_urssaf_covid(data: pd.DataFrame):
+    """
+    Règle experte
+    """
+
+    # compute change in social debt as a proportion of average cotisations over the past 12months
+    data["montant_part_ouvriere_latest"].fillna(0, inplace=True)
+    data["montant_part_patronale_latest"].fillna(0, inplace=True)
+    data["montant_part_ouvriere_july2020"].fillna(0, inplace=True)
+    data["montant_part_patronale_july2020"].fillna(0, inplace=True)
+    data["dette_sociale_july2020"] = (
+        data.montant_part_ouvriere_july2020 + data.montant_part_patronale_july2020
+    )
+    data["dette_sociale_latest"] = (
+        data.montant_part_ouvriere_latest + data.montant_part_patronale_latest
+    )
+
+    data["delta_dette"] = (
+        data.dette_sociale_latest - data.dette_sociale_july2020
+    ) / data.cotisation_moy12m_latest
+
+    tol = 0.2  # tolerate a change smaller than 20%
+
+    def rule(dataframe):
+        """
+        Expert rule to apply
+        """
+        value = dataframe["delta_dette"]
+        group = dataframe["group_final"]
+        if value > tol:
+            if group == "vert":
+                return "orange"
+            if group == "orange":
+                return "rouge"
+            if group == "rouge":
+                return "rouge"
+        return group
+
+    data["group_final_regle_urssaf"] = data.apply(rule, axis=1)
+
+    return data


Test patch:
diff --git a/tests/fake_data/dataframes.py b/tests/fake_data/dataframes.py
index a86536b..b88d574 100644
--- a/tests/fake_data/dataframes.py
+++ b/tests/fake_data/dataframes.py
@@ -26,3 +26,14 @@ df_test_code_naf = pd.DataFrame(
 )
 
 df_test_full = pd.concat((df_test_code_naf, df_test_acoss, df_test_paydex), axis=1)
+
+df_test_redressement_urssaf = pd.DataFrame(
+    {
+        "montant_part_ouvriere_latest": [50, 60, 100],
+        "montant_part_patronale_latest": [75, 50, 100],
+        "montant_part_ouvriere_july2020": [50, 50, 50],
+        "montant_part_patronale_july2020": [50, 50, 0],
+        "cotisation_moy12m_latest": [100, 100, 75],
+        "group_final": ["vert", "vert", "rouge"],
+    }
+)
diff --git a/tests/unit/pipelines_test.py b/tests/unit/pipelines_test.py
index 2dba9ba..9b62b1f 100644
--- a/tests/unit/pipelines_test.py
+++ b/tests/unit/pipelines_test.py
@@ -10,22 +10,23 @@ from predictsignauxfaibles.pipelines import (
 )
 
 from predictsignauxfaibles.preprocessors import Preprocessor
+from predictsignauxfaibles.redressements import Redressement
 
 from tests.fake_data.dataframes import df_test_full
 
 
 @pytest.mark.parametrize("pipeline", ALL_PIPELINES)
-def test_final_pipeline(pipeline):
-    for preprocessor in pipeline:
-        assert isinstance(preprocessor, Preprocessor)
-        assert hasattr(preprocessor, "name")
-        assert hasattr(preprocessor, "function")
-        assert hasattr(preprocessor, "input")
-        assert hasattr(preprocessor, "output")
-        assert isinstance(preprocessor.name, str)
-        assert callable(preprocessor.function)
-        assert isinstance(preprocessor.input, list)
-        assert isinstance(preprocessor.output, list) or preprocessor.output is None
+def test_all_pipeline(pipeline):
+    for step in pipeline:
+        assert isinstance(step, (Preprocessor, Redressement))
+        assert hasattr(step, "name")
+        assert hasattr(step, "function")
+        assert hasattr(step, "input")
+        assert hasattr(step, "output")
+        assert isinstance(step.name, str)
+        assert callable(step.function)
+        assert isinstance(step.input, list)
+        assert isinstance(step.output, list) or step.output is None
 
 
 def test_default_pipeline_sucess():
diff --git a/tests/unit/redressements_test.py b/tests/unit/redressements_test.py
new file mode 100644
index 0000000..2078717
--- /dev/null
+++ b/tests/unit/redressements_test.py
@@ -0,0 +1,11 @@
+# pylint: disable=missing-function-docstring
+import pandas as pd
+from predictsignauxfaibles.redressements import redressement_urssaf_covid
+
+from tests.fake_data.dataframes import df_test_redressement_urssaf
+
+
+def test_redressement_urssaf_covid():
+    data_in = df_test_redressement_urssaf.copy()
+    data_out = redressement_urssaf_covid(data_in)
+    assert (data_out["group_final_regle_urssaf"] == ["orange", "vert", "rouge"]).all()