Repository: ul-mds/gecko

Issue:
Empty target cells in replacement table mutators raises an error
E.g. given a file like:

```csv
source,target
"-",""
```

The `with_replacement_table` mutator will raise an error because it tries to use `NaN` as a substitution value.

```
Traceback (most recent call last):
  File "/home/mjugl/.local/share/JetBrains/Toolbox/apps/pycharm-professional/plugins/python-ce/helpers/pydev/pydevd.py", line 1557, in _exec
    pydev_imports.execfile(file, globals, locals)  # execute the script
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mjugl/.local/share/JetBrains/Toolbox/apps/pycharm-professional/plugins/python-ce/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/lobb_gecko/datagen.py", line 292, in <module>
    main()
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/lobb_gecko/datagen.py", line 240, in main
    df_mutated = mutator.mutate_data_frame(
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/.venv/lib/python3.12/site-packages/gecko/mutator.py", line 1648, in mutate_data_frame
    srs_mutated_lst = mutator([srs[mask_this_mutator] for srs in srs_columns])
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/.venv/lib/python3.12/site-packages/gecko/mutator.py", line 605, in _mutate
    return [_mutate_series(srs) for srs in srs_lst]
            ^^^^^^^^^^^^^^^^^^^
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/.venv/lib/python3.12/site-packages/gecko/mutator.py", line 596, in _mutate_series
    ].str.replace(source, target, n=1)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/.venv/lib/python3.12/site-packages/pandas/core/strings/accessor.py", line 137, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mjugl/PycharmProjects/lobb-gecko-analysis/.venv/lib/python3.12/site-packages/pandas/core/strings/accessor.py", line 1548, in replace
    raise TypeError("repl must be a string or callable")
TypeError: repl must be a string or callable
```

Discussion:

Solution patch:
diff --git a/gecko/generator.py b/gecko/generator.py
index 991ccf7..d975b15 100644
--- a/gecko/generator.py
+++ b/gecko/generator.py
@@ -153,6 +153,7 @@ def from_frequency_table(
         header=0 if header else None,  # header row index (`None` if not present)
         usecols=[value_column, freq_column],
         dtype={freq_column: "int", value_column: "str"},
+        keep_default_na=False,
         sep=delimiter,
         encoding=encoding,
     )
@@ -233,6 +234,7 @@ def from_multicolumn_frequency_table(
             freq_column: "int",
             **{value_column: "str" for value_column in value_columns},
         },
+        keep_default_na=False,
         sep=delimiter,
         encoding=encoding,
     )
diff --git a/gecko/mutator.py b/gecko/mutator.py
index a195490..d49f6d1 100644
--- a/gecko/mutator.py
+++ b/gecko/mutator.py
@@ -333,6 +333,7 @@ def __new_unknown_flag_error(flag: str):
         header=0 if header else None,
         dtype=str,
         usecols=[source_column, target_column, flags_column],
+        keep_default_na=False,
         sep=delimiter,
         encoding=encoding,
     )
@@ -524,6 +525,7 @@ def with_replacement_table(
         header=0 if header else None,
         dtype=str,
         usecols=[source_column, target_column],
+        keep_default_na=False,
         sep=delimiter,
         encoding=encoding,
     )
@@ -1072,12 +1074,13 @@ def with_categorical_values(
         header=0 if header else None,
         dtype=str,
         usecols=[value_column],
+        keep_default_na=False,
         sep=delimiter,
         encoding=encoding,
     )
 
     # fetch unique values
-    unique_values = pd.Series(df[value_column].dropna().unique())
+    unique_values = pd.Series(df[value_column].unique())
 
     def _mutate_series(srs: pd.Series) -> pd.Series:
         nonlocal unique_values
@@ -1445,7 +1448,13 @@ def with_regex_replacement_table(
     if rng is None:
         rng = np.random.default_rng()
 
-    df = pd.read_csv(csv_file_path, encoding=encoding, sep=delimiter, dtype=str)
+    df = pd.read_csv(
+        csv_file_path,
+        encoding=encoding,
+        keep_default_na=False,
+        sep=delimiter,
+        dtype=str,
+    )
 
     if pattern_column not in df.columns:
         raise ValueError(f"CSV file at `{csv_file_path}` doesn't have a pattern column")


Test patch:
diff --git a/tests/test_generator.py b/tests/test_generator.py
index b44c8d2..0fedd43 100644
--- a/tests/test_generator.py
+++ b/tests/test_generator.py
@@ -228,3 +228,36 @@ def test_to_dataframe(rng):
 
     for col in ["fruit", "type", "num"]:
         assert col in df.columns
+
+
+def test_from_frequency_table_nan(tmp_path, rng):
+    freq_file_path = tmp_path / "freq.csv"
+    freq_file_path.write_text('value,freq\n"",1\n"foobar",1\n')
+
+    gen_freq_table = generator.from_frequency_table(
+        freq_file_path,
+        value_column="value",
+        freq_column="freq",
+        rng=rng,
+    )
+
+    (srs,) = gen_freq_table(100)
+
+    assert pd.notna(srs).all()
+
+
+def test_from_multicolumn_frequency_table_nan(tmp_path, rng):
+    freq_file_path = tmp_path / "freq.csv"
+    freq_file_path.write_text('value1,value2,freq\n"","bar",1\n"foo","baz",1\n')
+
+    gen_freq_table = generator.from_multicolumn_frequency_table(
+        freq_file_path,
+        value_columns=["value1", "value2"],
+        freq_column="freq",
+        rng=rng,
+    )
+
+    (srs1, srs2) = gen_freq_table(100)
+
+    assert pd.notna(srs1).all()
+    assert pd.notna(srs2).all()
diff --git a/tests/test_mutator.py b/tests/test_mutator.py
index 13bda08..6992776 100644
--- a/tests/test_mutator.py
+++ b/tests/test_mutator.py
@@ -968,3 +968,73 @@ def test_mutate_data_frame_order_generates_different_results(rng_factory):
     )
 
     assert not (df_out_1["foo"] == df_out_2["foo"]).all()
+
+
+# see https://github.com/ul-mds/gecko/issues/69
+# CSV files containing empty cells will be evaluated to NaN, which causes this to raise an error
+def test_with_replacement_table_nan(tmp_path, rng):
+    replacement_table_file_path = tmp_path / "replacement.csv"
+    replacement_table_file_path.write_text('source,target\n"-",""\n')
+
+    mut_replace = with_replacement_table(
+        replacement_table_file_path,
+        source_column="source",
+        target_column="target",
+        inline=True,
+        rng=rng,
+    )
+
+    srs = pd.Series(["foo-bar", "foo-baz", "foo-bat"])
+    (srs_mut,) = mut_replace([srs])
+
+    assert (srs_mut == ["foobar", "foobaz", "foobat"]).all()
+
+
+def test_with_regex_replacement_table_nan(tmp_path, rng):
+    replacement_table_file_path = tmp_path / "replacement.csv"
+    replacement_table_file_path.write_text('pattern,dash\n".*(?P<dash>-).*",""\n')
+
+    mut_replace = with_regex_replacement_table(
+        replacement_table_file_path,
+        pattern_column="pattern",
+        rng=rng,
+    )
+
+    srs = pd.Series(["foo-bar", "foo-baz", "foo-bat"])
+    (srs_mut,) = mut_replace([srs])
+
+    assert (srs_mut == ["foobar", "foobaz", "foobat"]).all()
+
+
+def test_with_phonetic_replacement_table_nan(tmp_path, rng):
+    replacement_table_file_path = tmp_path / "replacement.csv"
+    replacement_table_file_path.write_text('source,target,flags\n"-","","_"\n')
+
+    mut_replace = with_phonetic_replacement_table(
+        replacement_table_file_path,
+        source_column="source",
+        target_column="target",
+        flags_column="flags",
+        rng=rng,
+    )
+
+    srs = pd.Series(["foo-bar", "foo-baz", "foo-bat"])
+    (srs_mut,) = mut_replace([srs])
+
+    assert (srs_mut == ["foobar", "foobaz", "foobat"]).all()
+
+
+def test_with_categorical_values_nan(tmp_path, rng):
+    cat_file_path = tmp_path / "gender.csv"
+    cat_file_path.write_text('gender\nf\n""\n')
+
+    mut_categorical = with_categorical_values(
+        cat_file_path,
+        value_column="gender",
+        rng=rng,
+    )
+
+    srs = pd.Series(["f"] * 10)
+    (srs_mut,) = mut_categorical([srs])
+
+    assert (srs_mut == "").all()