Repository: mathesar-foundation/mathesar

Issue:
Get filtered set of records from table
**Problem**
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->

At the moment, when we request records from a table, it's _all_ of the records or none.  We should be able to filter a way that supports retrieving "groups" of records, where a group is defined as a set of records where some subset of the columns match a set of values, equivalent to the SQL:

```sql
WHERE col1 = val1 AND col2 = val2 AND ... AND coln = valn
```

**Proposed solution**
<!-- A clear and concise description of your proposed solution or feature. -->

We should create a function that lets us filter string columns using the pattern above, and another function that retrieves the distinct tuples for a set of columns (i.e., the groups in this context).

**Additional context**
<!-- Add any other context or screenshots about the feature request here.-->

The interesting bit will be figuring out how to paginate the results, but without having to reperform the (costly) filtering query each time.


Discussion:

Resolving patch:
diff --git a/db/records.py b/db/records.py
--- a/db/records.py
+++ b/db/records.py
@@ -1,6 +1,9 @@
-from sqlalchemy import delete, select
+import logging
+from sqlalchemy import delete, select, and_, Column
 from sqlalchemy.inspection import inspect
 
+logger = logging.getLogger(__name__)
+
 
 def _get_primary_key_column(table):
     primary_key_list = list(inspect(table).primary_key)
@@ -18,7 +21,9 @@ def get_record(table, engine, id_value):
         return result[0] if result else None
 
 
-def get_records(table, engine, limit=None, offset=None, order_by=[]):
+def get_records(
+        table, engine, limit=None, offset=None, order_by=[], filters=[]
+):
     """
     Returns records from a table.
 
@@ -30,12 +35,72 @@ def get_records(table, engine, limit=None, offset=None, order_by=[]):
         order_by: list of SQLAlchemy ColumnElements to order by.  Should
                   usually be either a list of string column names, or a
                   list of columns from the given table.
+        filters:  list of tuples of type (ColumnElement, value), where
+                  ColumnElement is an SQLAlchemy ColumnElement, and value
+                  is a valid value for the associated column (i.e., the
+                  type must be correct)
     """
-    query = select(table).order_by(*order_by).limit(limit).offset(offset)
+    query = (
+        select(table)
+        .order_by(*order_by)
+        .limit(limit)
+        .offset(offset)
+        .where(_build_filter_conjunction(table, filters))
+    )
     with engine.begin() as conn:
         return conn.execute(query).fetchall()
 
 
+def _build_filter_conjunction(table, filters):
+    refined_filters = [
+        (table.columns[col] if type(col) == str else col, value)
+        for col, value in filters
+    ]
+    # We need a default of True (rather than empty), since invoking and_
+    # without arguments is deprecated.
+    return and_(True, *[col == value for col, value in refined_filters])
+
+
+def get_distinct_tuple_values(
+        column_list, engine, table=None, limit=None, offset=None,
+):
+    """
+    Returns distinct tuples from a given list of columns.
+
+    Args:
+        column_list: list of column names or SQLAlchemy column objects
+        engine:   SQLAlchemy engine object
+        table:    SQLAlchemy table object
+        limit:    int, gives number of rows to return
+        offset:   int, gives number of rows to skip
+
+    If no table is given, the column_list must consist entirely of
+    SQLAlchemy column objects associated with a table.
+    """
+    if table is not None:
+        column_objects = [
+            table.columns[col] if type(col) == str else col
+            for col in column_list
+        ]
+    else:
+        column_objects = column_list
+    try:
+        assert all([type(col) == Column for col in column_objects])
+    except AssertionError as e:
+        logger.error("All columns must be str or sqlalchemy.Column type")
+        raise e
+
+    query = (
+        select(*column_objects)
+        .distinct()
+        .limit(limit)
+        .offset(offset)
+    )
+    with engine.begin() as conn:
+        res = conn.execute(query).fetchall()
+    return [tuple(zip(column_objects, row)) for row in res]
+
+
 def create_record_or_records(table, engine, record_data):
     """
     record_data can be a dictionary, tuple, or list of dictionaries or tuples.


Test patch:
diff --git a/db/tests/test_records.py b/db/tests/test_records.py
--- a/db/tests/test_records.py
+++ b/db/tests/test_records.py
@@ -80,3 +80,181 @@ def test_get_records_orders_before_limiting(roster_table_obj):
         roster, engine, limit=1, order_by=["Grade", "Student Name"]
     )
     assert record_list[0][7] == 25 and record_list[0][2] == "Amy Gamble"
+
+
+def test_get_records_filters_using_col_str_names(roster_table_obj):
+    roster, engine = roster_table_obj
+    filter_list = [("Student Name", "Amy Gamble"), ("Subject", "Math")]
+    record_list = records.get_records(
+        roster, engine, filters=filter_list
+    )
+    assert all(
+        [
+            len(record_list) == 1,
+            record_list[0][2] == "Amy Gamble",
+            record_list[0][6] == "Math",
+        ]
+    )
+
+
+def test_get_records_filters_using_col_objects(roster_table_obj):
+    roster, engine = roster_table_obj
+    filter_list = [
+        (roster.columns["Student Name"], "Amy Gamble"),
+        (roster.columns["Subject"], "Math"),
+    ]
+    record_list = records.get_records(
+        roster, engine, filters=filter_list
+    )
+    assert all(
+        [
+            len(record_list) == 1,
+            record_list[0][2] == "Amy Gamble",
+            record_list[0][6] == "Math",
+        ]
+    )
+
+
+def test_get_records_filters_using_mixed_col_objects_and_str(roster_table_obj):
+    roster, engine = roster_table_obj
+    filter_list = [
+        (roster.columns["Student Name"], "Amy Gamble"),
+        ("Subject", "Math"),
+    ]
+    record_list = records.get_records(
+        roster, engine, filters=filter_list
+    )
+    assert all(
+        [
+            len(record_list) == 1,
+            record_list[0][2] == "Amy Gamble",
+            record_list[0][6] == "Math",
+        ]
+    )
+
+
+def test_get_records_filters_with_numeric_col(roster_table_obj):
+    roster, engine = roster_table_obj
+    filter_list = [
+        (roster.columns["Student Name"], "Amy Gamble"),
+        (roster.columns["Grade"], 74),
+    ]
+    record_list = records.get_records(
+        roster, engine, filters=filter_list
+    )
+    assert all(
+        [
+            len(record_list) == 1,
+            record_list[0][2] == "Amy Gamble",
+            record_list[0][7] == 74,
+        ]
+    )
+
+
+def test_get_records_filters_with_miss(roster_table_obj):
+    roster, engine = roster_table_obj
+    filter_list = [
+        (roster.columns["Student Name"], "Amy Gamble"),
+        (roster.columns["Grade"], 75),
+    ]
+    record_list = records.get_records(
+        roster, engine, filters=filter_list
+    )
+    assert len(record_list) == 0
+
+
+def test_get_distinct_tuple_values_length(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        roster.columns["Student Number"],
+        roster.columns["Student Email"],
+    ]
+    record_list = records.get_distinct_tuple_values(
+        column_list, engine
+    )
+    assert len(record_list) == 259
+
+
+def test_get_distinct_tuple_values_distinct(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        roster.columns["Student Number"],
+        roster.columns["Student Email"],
+    ]
+    record_list = records.get_distinct_tuple_values(
+        column_list, engine
+    )
+    for i in range(len(record_list) - 1):
+        assert record_list[i] != record_list[i + 1]
+
+
+def test_get_distinct_tuple_values_raises_when_no_table(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        "Student Number",
+        "Student Email",
+    ]
+    with pytest.raises(AssertionError):
+        records.get_distinct_tuple_values(
+            column_list, engine
+        )
+
+
+def test_get_distinct_tuple_values_with_string_column_input(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        "Student Number",
+        "Student Email",
+    ]
+    record_list = records.get_distinct_tuple_values(
+        column_list, engine, table=roster,
+    )
+    assert len(record_list) == 259
+
+
+def test_get_distinct_tuple_values_limit(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        "Student Number",
+        "Student Email",
+    ]
+    record_list = records.get_distinct_tuple_values(
+        column_list, engine, table=roster, limit=10
+    )
+    assert len(record_list) == 10
+
+
+def test_get_distinct_tuple_values_offset(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        "Student Number",
+        "Student Email",
+    ]
+    record_list_base = records.get_distinct_tuple_values(
+        column_list, engine, table=roster, limit=20
+    )
+    record_list_offset = records.get_distinct_tuple_values(
+        column_list, engine, table=roster, limit=10, offset=10
+    )
+    assert record_list_offset == record_list_base[10:]
+
+
+def test_get_distinct_tuple_values_feeds_get_records(roster_table_obj):
+    roster, engine = roster_table_obj
+    column_list = [
+        "Student Number",
+        "Student Email",
+    ]
+    distinct_tuples = records.get_distinct_tuple_values(
+        column_list, engine, table=roster, limit=2
+    )
+    record_list = records.get_records(
+        roster, engine, filters=distinct_tuples[0]
+    )
+    assert all(
+        [
+            record[1] == distinct_tuples[0][0][1]
+            and record[3] == distinct_tuples[0][1][1]
+            for record in record_list
+        ]
+    )