Repository: mathesar-foundation/mathesar

Issue:
We need an "is not empty" filter
## Problem
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->
We have an "is empty" filter for all data types, but there's no way to filter on "is not empty".

## Proposed solution
<!-- A clear and concise description of your proposed solution or feature. -->
We should have a new "is not empty" filter.


Discussion:
Should we instead have a "not" modifier that can be applied to all filters?
I do think that's a better solution in the long run. 

It wouldn't currently work for the Data Explorer because we don't have modifiers for filters (and that's what I need it for). I think implementing modifiers for queries would take a lot longer than just adding an "is not empty" filter, but @pavish and @dmos62 would know better.
@seancolsen filtering API is a subset of the function API; while filtering doesn't support modifiers (like Kriti pointed out), the functions API does. So far we've been solving these kinds of problems by creating a filter for each variation a user might want. If or when we decide that we need a more powerful filtering/transformation UI, backend is already in an ok position to support that.
> Should we instead have a "not" modifier that can be applied to all filters?

It is not a technical issue, rather a design problem which needs to be solved. Our design does not take modifiers or multiple and/or nesting into account. This will take considerable time, and we would want it to be uniform across the product, for both tables and DE.

At the moment, for this issue, I'd like to only add the 'is not empty' filter to the existing filters list.

Resolving patch:
diff --git a/db/functions/base.py b/db/functions/base.py
--- a/db/functions/base.py
+++ b/db/functions/base.py
@@ -170,9 +170,9 @@ def to_sa_expression(*items):
         return list(items)
 
 
-class Empty(DBFunction):
-    id = 'empty'
-    name = 'is empty'
+class Null(DBFunction):
+    id = 'null'
+    name = 'is null'
     hints = tuple([
         hints.returns(hints.boolean),
         hints.parameter_count(1),
diff --git a/db/functions/operations/deserialize.py b/db/functions/operations/deserialize.py
--- a/db/functions/operations/deserialize.py
+++ b/db/functions/operations/deserialize.py
@@ -9,7 +9,7 @@ def get_db_function_from_ma_function_spec(spec):
 
     ```
     {"and": [
-        {"empty": [
+        {"null": [
             {"column_name": ["some_column"]},
         ]},
         {"equal": [
diff --git a/db/functions/packed.py b/db/functions/packed.py
--- a/db/functions/packed.py
+++ b/db/functions/packed.py
@@ -30,6 +30,23 @@ def unpack(self):
         pass
 
 
+class NotNull(DBFunctionPacked):
+    id = 'not_null'
+    name = 'Is not null'
+    hints = tuple([
+        hints.returns(hints.boolean),
+        hints.parameter_count(1),
+        hints.parameter(0, hints.any),
+        hints.mathesar_filter,
+    ])
+
+    def unpack(self):
+        param0 = self.parameters[0]
+        return base.Not([
+            base.Null([param0]),
+        ])
+
+
 class LesserOrEqual(DBFunctionPacked):
     id = 'lesser_or_equal'
     name = 'is lesser or equal to'


Test patch:
diff --git a/db/tests/functions/operations/test_deserialize.py b/db/tests/functions/operations/test_deserialize.py
--- a/db/tests/functions/operations/test_deserialize.py
+++ b/db/tests/functions/operations/test_deserialize.py
@@ -14,7 +14,7 @@
         UnknownDBFunctionID
     ),
     (
-        {"empty": {"column_name": ["varchar"]}, },
+        {"null": {"column_name": ["varchar"]}, },
         BadDBFunctionFormat
     ),
 ]
diff --git a/db/tests/functions/operations/test_filter.py b/db/tests/functions/operations/test_filter.py
--- a/db/tests/functions/operations/test_filter.py
+++ b/db/tests/functions/operations/test_filter.py
@@ -4,10 +4,10 @@
 from db.utils import execute_pg_query
 
 from db.functions.base import (
-    ColumnName, Not, Literal, Empty, Equal, Greater, And, Or, StartsWith, Contains, StartsWithCaseInsensitive, ContainsCaseInsensitive
+    ColumnName, Not, Literal, Null, Equal, Greater, And, Or, StartsWith, Contains, StartsWithCaseInsensitive, ContainsCaseInsensitive
 )
 from db.functions.packed import (
-    GreaterOrEqual, LesserOrEqual
+    GreaterOrEqual, LesserOrEqual, NotNull
 )
 from db.functions.operations.apply import apply_db_function_as_filter
 
@@ -21,8 +21,8 @@ def _ilike(x, v):
 
 
 database_functions = {
-    "is_null": lambda x: Empty([ColumnName([x])]),
-    "is_not_null": lambda x: Not([Empty([ColumnName([x])])]),
+    "is_null": lambda x: Null([ColumnName([x])]),
+    "is_not_null": lambda x: NotNull([ColumnName([x])]),
     "eq": lambda x, v: Equal([ColumnName([x]), Literal([v])]),
     "gt": lambda x, v: Greater([ColumnName([x]), Literal([v])]),
     "and": lambda x: And(x),
diff --git a/db/tests/queries/test_base.py b/db/tests/queries/test_base.py
--- a/db/tests/queries/test_base.py
+++ b/db/tests/queries/test_base.py
@@ -172,7 +172,7 @@ def test_DBQuery_all_sa_columns_map_summarized_columns(engine_with_library):
             spec={"lesser": [{"column_name": ["Due Date"]}, {"literal": ["2022-08-10"]}]}
         ),
         tbase.Filter(
-            spec={"empty": [{"column_name": ["Check In Time"]}]}
+            spec={"null": [{"column_name": ["Check In Time"]}]}
         ),
         tbase.Summarize(
             spec={
diff --git a/mathesar/tests/api/query/test_library_demo_reports.py b/mathesar/tests/api/query/test_library_demo_reports.py
--- a/mathesar/tests/api/query/test_library_demo_reports.py
+++ b/mathesar/tests/api/query/test_library_demo_reports.py
@@ -72,7 +72,7 @@ def create_overdue_books_query(library_ma_tables, get_uid, client):
             },
             {
                 "spec": {
-                    "empty": [{"column_name": ["Check In Time"]}]
+                    "null": [{"column_name": ["Check In Time"]}]
                 },
                 "type": "filter"
             },
diff --git a/mathesar/tests/api/test_record_api.py b/mathesar/tests/api/test_record_api.py
--- a/mathesar/tests/api/test_record_api.py
+++ b/mathesar/tests/api/test_record_api.py
@@ -215,10 +215,10 @@ def test_filter_with_added_columns(create_patents_table, client):
             lambda new_column_id, value: {"equal": [{"column_id": [new_column_id]}, {"literal": [value]}]},
             False, 2),
         (
-            lambda new_column_id, _: {"empty": [{"column_id": [new_column_id]}]},
+            lambda new_column_id, _: {"null": [{"column_id": [new_column_id]}]},
             None, 1394),
         (
-            lambda new_column_id, _: {"not": [{"empty": [{"column_id": [new_column_id]}]}]},
+            lambda new_column_id, _: {"not": [{"null": [{"column_id": [new_column_id]}]}]},
             None, 49),
     ]
 
@@ -1182,7 +1182,7 @@ def test_record_list_filter_exceptions(create_patents_table, client):
     table_name = f"NASA Record List {exception.__name__}"
     table = create_patents_table(table_name)
     columns_name_id_map = table.get_column_name_id_bidirectional_map()
-    filter_list = json.dumps({"empty": [{"column_name": [columns_name_id_map['Center']]}]})
+    filter_list = json.dumps({"null": [{"column_name": [columns_name_id_map['Center']]}]})
     with patch.object(DBQuery, "get_records", side_effect=exception):
         response = client.get(
             f'/api/db/v0/tables/{table.id}/records/?filters={filter_list}'
diff --git a/mathesar/tests/filters/test_filters.py b/mathesar/tests/filters/test_filters.py
--- a/mathesar/tests/filters/test_filters.py
+++ b/mathesar/tests/filters/test_filters.py
@@ -9,7 +9,8 @@ def test_available_filters_structure(engine_with_schema):
     some_filters_that_we_expect_to_be_there = [
         'greater',
         'lesser',
-        'empty',
+        'null',
+        'not_null',
         'equal',
         'greater_or_equal',
         'contains_case_insensitive',