Repository: mathesar-foundation/mathesar

Issue:
DB function to get rows with duplicate values
## Problem
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->
The ["Working with Columns" design spec](https://wiki.mathesar.org/en/design/specs/working-with-columns) adds a filter for duplicate values for a column, to assist users with resolving non-unique values when they want to set a unique constraint for a column.

## Proposed solution
<!-- A clear and concise description of your proposed solution or feature. -->
We need to add a way to get columns with duplicate values in our `db` module. Please see #501 for relevant discussion about how to achieve this.

## Additional context
<!-- Add any other context or screenshots about the feature request here.-->
- API issue: #501
- Frontend issue: #499
- Design issue: #324 
- Design discussion: #436


Discussion:

Resolving patch:
diff --git a/db/records.py b/db/records.py
--- a/db/records.py
+++ b/db/records.py
@@ -1,12 +1,17 @@
 import logging
-from sqlalchemy import delete, select, Column, func
+from sqlalchemy import delete, select, Column, func, true, and_
 from sqlalchemy.inspection import inspect
 from sqlalchemy_filters import apply_filters, apply_sort
-from sqlalchemy_filters.exceptions import FieldNotFound
+from sqlalchemy_filters.exceptions import (
+    FieldNotFound, BadFilterFormat, FilterFieldNotFound
+)
 
 
 logger = logging.getLogger(__name__)
 
+IS_DUPE = "_is_dupe"
+CONJUNCTIONS = ("and", "or", "not")
+
 
 # Grouping exceptions follow the sqlalchemy_filters exceptions patterns
 class BadGroupFormat(Exception):
@@ -31,8 +36,14 @@ def _create_col_objects(table, column_list):
     ]
 
 
-def _get_query(table, limit, offset, order_by, filters):
-    query = select(table).limit(limit).offset(offset)
+def _get_query(table, limit, offset, order_by, filters, cols=None):
+    dupe_columns, filters = _get_dupe_columns(table, filters)
+    if dupe_columns:
+        query = _create_query_with_dupe(table, dupe_columns, cols)
+    else:
+        query = select(*(cols or table.c)).select_from(table)
+
+    query = query.limit(limit).offset(offset)
     if order_by is not None:
         query = apply_sort(query, order_by)
     if filters is not None:
@@ -46,6 +57,58 @@ def _execute_query(query, engine):
         return records
 
 
+def _get_dupe_columns(table, filters):
+    try:
+        get_dupe_ops = [f for f in filters if f.get("op") == "get_duplicates"]
+        non_get_dupe_ops = [f for f in filters if f.get("op") != "get_duplicates"]
+    except AttributeError:
+        # Ignore formatting errors - they will be handled by sqlalchemy_filters
+        return None, filters
+
+    _validate_nested_ops(non_get_dupe_ops)
+    if len(get_dupe_ops) > 1:
+        raise BadFilterFormat("get_duplicates can only be specified a single time")
+    elif len(get_dupe_ops) == 1:
+        dupe_cols = get_dupe_ops[0]['value']
+        for col in dupe_cols:
+            if col not in table.c:
+                raise FilterFieldNotFound(f"Table {table.name} has no column `{col}`.")
+        return get_dupe_ops[0]['value'], non_get_dupe_ops
+    else:
+        return None, filters
+
+
+def _validate_nested_ops(filters):
+    for op in filters:
+        if op.get("op") == "get_duplicates":
+            raise BadFilterFormat("get_duplicates can not be nested")
+        for field in CONJUNCTIONS:
+            if field in op:
+                _validate_nested_ops(op[field])
+
+
+def _create_query_with_dupe(table, dupe_columns, cols=None):
+    subq_table = table.alias()
+    table_dupe_columns = [c for c in table.c if c.name in dupe_columns]
+    subq_dupe_columns = [c for c in subq_table.c if c.name in dupe_columns]
+    subq = (
+        select(true().label(IS_DUPE))
+        .select_from(subq_table)
+        .group_by(*subq_dupe_columns)
+        .having(and_(
+            func.count() > 1,
+            *[c == subq_c for c, subq_c in zip(table_dupe_columns, subq_dupe_columns)]
+        ))
+        .lateral("lateral_subq")
+    )
+    query = (
+        select(*(cols or table.c))
+        .select_from(table.join(subq, true()))
+        .where(subq.c[IS_DUPE])
+    )
+    return query
+
+
 def get_record(table, engine, id_value):
     primary_key_column = _get_primary_key_column(table)
     query = select(table).where(primary_key_column == id_value)
diff --git a/db/tables.py b/db/tables.py
--- a/db/tables.py
+++ b/db/tables.py
@@ -6,13 +6,12 @@
 )
 from sqlalchemy.schema import DDLElement, DropTable
 from sqlalchemy.ext import compiler
-from sqlalchemy_filters import apply_filters
 from alembic.migration import MigrationContext
 from alembic.operations import Operations
 from sqlalchemy.exc import NoSuchTableError, InternalError
 from psycopg2.errors import DependentObjectsStillExist
 
-from db import columns, constants, schemas
+from db import columns, constants, schemas, records
 from db.types import alteration, inference
 
 
@@ -456,11 +455,10 @@ def create_empty_table(name):
 
 
 def get_count(table, engine, filters=[]):
-    query = select([func.count()]).select_from(table)
-    if filters is not None:
-        query = apply_filters(query, filters)
-    with engine.begin() as conn:
-        return conn.execute(query).scalar()
+    col_name = "_count"
+    cols = [func.count().label(col_name)]
+    query = records._get_query(table, None, None, None, filters, cols)
+    return records._execute_query(query, engine)[0][col_name]
 
 
 def update_table_column_types(schema, table_name, engine):


Test patch:
diff --git a/db/tests/records/test_filtering.py b/db/tests/records/test_filtering.py
--- a/db/tests/records/test_filtering.py
+++ b/db/tests/records/test_filtering.py
@@ -1,6 +1,7 @@
 import re
 import pytest
 from datetime import datetime
+from collections import Counter
 
 from sqlalchemy_filters.exceptions import BadFilterFormat, FilterFieldNotFound
 
@@ -73,6 +74,24 @@ def test_get_records_filters_with_miss(roster_table_obj):
     assert len(record_list) == 0
 
 
+def test_get_records_filters_duplicates(roster_table_obj):
+    roster, engine = roster_table_obj
+    dupe_cols = ["Grade", "Subject"]
+    filter_list = [{"field": "", "op": "get_duplicates", "value": dupe_cols}]
+
+    full_record_list = records.get_records(roster, engine)
+    dupe_record_list = records.get_records(roster, engine, filters=filter_list)
+
+    # Ensures that:
+    #   - All duplicate values in the table appeared in our query
+    #   - All values in our query are duplicate values
+    #   - All duplicate values appear the correct number of times
+    all_counter = Counter(tuple(r[c] for c in dupe_cols) for r in full_record_list)
+    all_counter = {k: v for k, v in all_counter.items() if v > 1}
+    got_counter = Counter(tuple(r[c] for c in dupe_cols) for r in dupe_record_list)
+    assert all_counter == got_counter
+
+
 def _like(x, v):
     return re.match(v.replace("%", ".*"), x) is not None
 
@@ -270,6 +289,16 @@ def test_get_records_filters_nested_boolean_ops(filter_sort_table_obj):
     ([{"not": []}], BadFilterFormat),
     ([{"not": [{"field": "date", "op": "is_null"} for _ in range(2)]}], BadFilterFormat),
     ([{"field": "non_existent", "op": "eq", "value": "test"}], FilterFieldNotFound),
+    ([
+        {"field": "date", "op": "get_duplicates", "value": ["non_existent"]},
+    ], FilterFieldNotFound),
+    ([
+        {"field": "", "op": "get_duplicates", "value": ["date"]},
+        {"field": "", "op": "get_duplicates", "value": ["date"]}
+    ], BadFilterFormat),
+    ([
+        {"not": [{"field": "", "op": "get_duplicates", "value": "date"}]}
+    ], BadFilterFormat),
 ]
 
 
diff --git a/mathesar/tests/views/api/test_record_api.py b/mathesar/tests/views/api/test_record_api.py
--- a/mathesar/tests/views/api/test_record_api.py
+++ b/mathesar/tests/views/api/test_record_api.py
@@ -75,7 +75,7 @@ def test_record_list_filter(create_table, client):
 
     with patch.object(
         records, "get_records", side_effect=records.get_records
-    ) as mock_infer:
+    ) as mock_get:
         response = client.get(
             f'/api/v0/tables/{table.id}/records/?filters={json_filter_list}'
         )
@@ -84,8 +84,23 @@ def test_record_list_filter(create_table, client):
     assert response.status_code == 200
     assert response_data['count'] == 2
     assert len(response_data['results']) == 2
-    assert mock_infer.call_args is not None
-    assert mock_infer.call_args[1]['filters'] == filter_list
+    assert mock_get.call_args is not None
+    assert mock_get.call_args[1]['filters'] == filter_list
+
+
+def test_record_list_filter_duplicates(create_table, client):
+    table_name = 'NASA Record List Filter Duplicates'
+    table = create_table(table_name)
+
+    filter_list = [
+        {'field': '', 'op': 'get_duplicates', 'value': ['Patent Expiration Date']}
+    ]
+    json_filter_list = json.dumps(filter_list)
+
+    with patch.object(records, "get_records") as mock_get:
+        client.get(f'/api/v0/tables/{table.id}/records/?filters={json_filter_list}')
+    assert mock_get.call_args is not None
+    assert mock_get.call_args[1]['filters'] == filter_list
 
 
 def test_record_list_sort(create_table, client):
@@ -100,7 +115,7 @@ def test_record_list_sort(create_table, client):
 
     with patch.object(
         records, "get_records", side_effect=records.get_records
-    ) as mock_infer:
+    ) as mock_get:
         response = client.get(
             f'/api/v0/tables/{table.id}/records/?order_by={json_order_by}'
         )
@@ -110,8 +125,8 @@ def test_record_list_sort(create_table, client):
     assert response_data['count'] == 1393
     assert len(response_data['results']) == 50
 
-    assert mock_infer.call_args is not None
-    assert mock_infer.call_args[1]['order_by'] == order_by
+    assert mock_get.call_args is not None
+    assert mock_get.call_args[1]['order_by'] == order_by
 
 
 def _test_record_list_group(table, client, group_count_by, expected_groups):
@@ -125,7 +140,7 @@ def _test_record_list_group(table, client, group_count_by, expected_groups):
 
     with patch.object(
         records, "get_group_counts", side_effect=records.get_group_counts
-    ) as mock_infer:
+    ) as mock_get:
         response = client.get(f'/api/v0/tables/{table.id}/records/?{query_str}')
         response_data = response.json()
 
@@ -144,8 +159,8 @@ def _test_record_list_group(table, client, group_count_by, expected_groups):
     for expected_group in expected_groups:
         assert expected_group in returned_groups
 
-    assert mock_infer.call_args is not None
-    assert mock_infer.call_args[0][2] == group_count_by
+    assert mock_get.call_args is not None
+    assert mock_get.call_args[0][2] == group_count_by
 
 
 def test_record_list_group_single_column(create_table, client):