Repository: mathesar-foundation/mathesar

Issue:
Implement filtering options for date & time types
## Problem
- We need to ensure that records that include columns of all Date & Time types support the following filters via API: 
  - between {x} and {y}
  - is {x}
  - is not {x}
  - before {x}
  - after {x}
  - on or before {x}
  - or or after {x}
  - is empty
  - is not empty
- We need to also ensure that values of the filters don't have to be an exact date. We should accept natural language like "next month", or "tomorrow".
  - We could use https://dateparser.readthedocs.io/

This involves:
- Implementing the filters in the backend
- Updating the `/api/v0/databases/<id>/types/` endpoint to store available filters on this type
  - Filter information should include the number of parameters needing to be passed in (e.g. `between` needs 2 parameters, `is empty` needs 0)

## Additional context
- We're using our fork of `sqlalchemy-filters` to provide filtering. See: https://github.com/centerofci/sqlalchemy-filters.
- This issue is blocked by implementation of Date & Time types: 
  - #424 
  - #425 
  - #426
- #557 provides some context on storing filters.
Implement filtering options for duration type
## Problem
- We need to ensure that records that include columns of duration types support the following filters via API: 
  - between {x} and {y}
  - equals {x}
  - does not equal {x}
  - greater than {x}
  - less than {x}
  - greater than or equals {x}
  - less than or equals  {x}
  - is empty
  - is not empty
- We need to also ensure that values of the filters don't have to be a number. We should accept natural language like "an hour", or "2 days".
  - We could use something like https://github.com/oleiade/durations or https://github.com/wroberts/pytimeparse

This involves:
- Implementing the filters in the backend
- Updating the `/api/v0/databases/<id>/types/` endpoint to store available filters on this type
  - Filter information should include the number of parameters needing to be passed in (e.g. `between` needs 2 parameters, `is empty` needs 0)

## Additional context
- These filters are the same as those for Number types. We should just reuse those.
  - The only additional thing we need to implement is converting natural language to numbers.
  - Relevant issue: #385 
- We're using our fork of `sqlalchemy-filters` to provide filtering. See: https://github.com/centerofci/sqlalchemy-filters.
- This issue is blocked by implementation of #430  
- #557 provides some context on storing filters.

Marking as blocked until #385 and #430 are complete.


Discussion:

Similarly to #426, I think there are advantages to using the native PostgreSQL parsing for the human-friendly duration strings, and it's already quite sophisticated and complete.

Resolving patch:
diff --git a/db/types/base.py b/db/types/base.py
--- a/db/types/base.py
+++ b/db/types/base.py
@@ -156,8 +156,10 @@ def _add_to_db_type_hintsets(db_types, hints):
 
     # string-like types get the "string_like" hint
     string_like_db_types = (
-        PostgresType.CHARACTER_VARYING,
+        PostgresType.CHAR,
         PostgresType.CHARACTER,
+        PostgresType.CHARACTER_VARYING,
+        PostgresType.NAME,
         PostgresType.TEXT,
         MathesarCustomType.URI,
         MathesarCustomType.EMAIL,
@@ -175,10 +177,25 @@ def _add_to_db_type_hintsets(db_types, hints):
         PostgresType.SMALLINT,
         PostgresType.NUMERIC,
         PostgresType.REAL,
+        PostgresType.MONEY,
     )
     hints_for_numeric_db_types = (hints.comparable,)
     _add_to_db_type_hintsets(numeric_db_types, hints_for_numeric_db_types)
 
+    # time related types get the "comparable" hint
+    time_related_db_types = (
+        PostgresType.DATE,
+        PostgresType.TIME,
+        PostgresType.TIME_WITH_TIME_ZONE,
+        PostgresType.TIME_WITHOUT_TIME_ZONE,
+        PostgresType.TIMESTAMP,
+        PostgresType.TIMESTAMP_WITH_TIME_ZONE,
+        PostgresType.TIMESTAMP_WITHOUT_TIME_ZONE,
+        PostgresType.INTERVAL,
+    )
+    hints_for_time_related_types = (hints.comparable,)
+    _add_to_db_type_hintsets(time_related_db_types, hints_for_time_related_types)
+
     return frozendict(db_types_hinted)
 
 


Test patch:
diff --git a/db/tests/conftest.py b/db/tests/conftest.py
--- a/db/tests/conftest.py
+++ b/db/tests/conftest.py
@@ -16,6 +16,7 @@
 RESOURCES = os.path.join(FILE_DIR, "resources")
 ROSTER_SQL = os.path.join(RESOURCES, "roster_create.sql")
 URIS_SQL = os.path.join(RESOURCES, "uris_create.sql")
+TIMES_SQL = os.path.join(RESOURCES, "times_create.sql")
 FILTER_SORT_SQL = os.path.join(RESOURCES, "filter_sort_create.sql")
 
 
@@ -29,6 +30,9 @@ def engine_with_schema(engine):
         conn.execute(text(f"DROP SCHEMA {schema} CASCADE;"))
 
 
+# TODO tech debt: notice that below fixtures are very similar
+
+
 @pytest.fixture
 def engine_with_roster(engine_with_schema):
     engine, schema = engine_with_schema
@@ -55,6 +59,19 @@ def engine_with_uris(engine_with_schema):
         conn.execute(DropSchema(base.SCHEMA, cascade=True, if_exists=True))
 
 
+@pytest.fixture
+def engine_with_times(engine_with_schema):
+    engine, schema = engine_with_schema
+    _add_custom_types_to_engine(engine)
+    install.install_mathesar_on_database(engine)
+    with engine.begin() as conn, open(TIMES_SQL) as f:
+        conn.execute(text(f"SET search_path={schema}"))
+        conn.execute(text(f.read()))
+    yield engine, schema
+    with engine.begin() as conn:
+        conn.execute(DropSchema(base.SCHEMA, cascade=True, if_exists=True))
+
+
 @pytest.fixture
 def engine_with_filter_sort(engine_with_schema):
     engine, schema = engine_with_schema
@@ -114,9 +131,17 @@ def extracted_remainder_roster(engine_with_roster, roster_table_name, roster_ext
     return teachers, roster_no_teachers, roster, engine, schema
 
 
+@pytest.fixture
+def times_table_obj(engine_with_times):
+    engine, schema = engine_with_times
+    metadata = MetaData(bind=engine)
+    table = Table("times", metadata, schema=schema, autoload_with=engine)
+    return table, engine
+
+
 @pytest.fixture
 def roster_table_obj(engine_with_roster, roster_table_name):
     engine, schema = engine_with_roster
     metadata = MetaData(bind=engine)
-    roster = Table(roster_table_name, metadata, schema=schema, autoload_with=engine)
-    return roster, engine
+    table = Table(roster_table_name, metadata, schema=schema, autoload_with=engine)
+    return table, engine
diff --git a/db/tests/functions/operations/test_filter.py b/db/tests/functions/operations/test_filter.py
--- a/db/tests/functions/operations/test_filter.py
+++ b/db/tests/functions/operations/test_filter.py
@@ -191,6 +191,25 @@ def test_filtering_nested_boolean_ops(filter_sort_table_obj):
                 and (record.varchar == "string42" or record.numeric == 24))
 
 
+@pytest.mark.parametrize("column_name,main_db_function,literal_param,expected_count", [
+    ("time", Equal, "06:05:06.789", 1),
+    ("date", Equal, "1999-01-08", 1),
+    ("time", LesserOrEqual, "04:05:06.789", 2),
+    ("timestamp", Greater, "1995-01-08", 1),
+    ("interval", Greater, "P1Y2M3DT4H5M6S", 2),
+])
+def test_filtering_time_types(times_table_obj, column_name, main_db_function, literal_param, expected_count):
+    table, engine = times_table_obj
+    selectable = table.select()
+    db_function = main_db_function([
+        ColumnName([column_name]),
+        Literal([literal_param]),
+    ])
+    query = apply_db_function_as_filter(selectable, db_function)
+    record_list = execute_query(engine, query)
+    assert len(record_list) == expected_count
+
+
 @pytest.mark.parametrize("column_name,main_db_function,literal_param,expected_count", [
     ("Student Name", StartsWithCaseInsensitive, "stephanie", 15),
     ("Student Name", StartsWith, "stephanie", 0),
diff --git a/db/tests/resources/times_create.sql b/db/tests/resources/times_create.sql
new file mode 100644
--- /dev/null
+++ b/db/tests/resources/times_create.sql
@@ -0,0 +1,29 @@
+CREATE TABLE "times" (
+    id integer NOT NULL,
+    "time" time,
+    "date" date,
+    "timestamp" timestamp,
+    "interval" interval
+);
+
+CREATE SEQUENCE "times_id_seq"
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE "times_id_seq" OWNED BY "times".id;
+
+ALTER TABLE ONLY "times" ALTER COLUMN id SET DEFAULT nextval('"times_id_seq"'::regclass);
+
+INSERT INTO "times" VALUES
+(1, '04:05:06.789', '1999-01-08', '1999-01-08 04:05:06 -8:00', 'P1Y2M3DT4H5M6S'),
+(2, '06:05:06.789', '2010-01-08', '1980-01-08 04:05:06 -8:00', 'P5Y2M3DT4H5M6S'),
+(3, '01:05:06.789', '2013-01-08', '1981-01-08 04:05:06 -8:00', 'P3Y5M3DT4H5M6S');
+
+SELECT pg_catalog.setval('"times_id_seq"', 1000, true);
+
+ALTER TABLE ONLY "times"
+    ADD CONSTRAINT "times_pkey" PRIMARY KEY (id);