Repository: mathesar-foundation/mathesar

Issue:
Create canonical ordering for tables and return it by default
## Problem
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->
We should have a canonical ordering for each table whenever possible. This will make infinite scroll easier to deal with.

## Proposed solution
<!-- A clear and concise description of your proposed solution or feature. -->
We should order by primary key by default if the table has one. Otherwise, we should use the entire row as a sorting key (it won't be possible to tell the difference if two identical rows "switch order"). We should always return rows in these orders when they are accessed unless the user specifies a different `ORDER BY`.

## Additional context
<!-- Add any other context or screenshots about the feature request here.-->
- Please see #361


Discussion:
Be aware, using the entire row as a sorting key will be performance-degrading at scale, so we'll eventually need to revisit.

Resolving patch:
diff --git a/db/records.py b/db/records.py
--- a/db/records.py
+++ b/db/records.py
@@ -72,6 +72,17 @@ def get_records(
                   field, in addition to an 'value' field if appropriate.
                   See: https://github.com/centerofci/sqlalchemy-filters#filters-format
     """
+    if not order_by:
+        # Set default ordering if none was requested
+        if len(table.primary_key.columns) > 0:
+            # If there are primary keys, order by all primary keys
+            order_by = [{'field': col, 'direction': 'asc'}
+                        for col in table.primary_key.columns]
+        else:
+            # If there aren't primary keys, order by all columns
+            order_by = [{'field': col, 'direction': 'asc'}
+                        for col in table.columns]
+
     query = _get_query(table, limit, offset, order_by, filters)
     return _execute_query(query, engine)
 


Test patch:
diff --git a/db/tests/records/test_sorting.py b/db/tests/records/test_sorting.py
--- a/db/tests/records/test_sorting.py
+++ b/db/tests/records/test_sorting.py
@@ -1,5 +1,7 @@
 import pytest
 
+from sqlalchemy import MetaData, Table
+from sqlalchemy.schema import DropConstraint
 from sqlalchemy_filters.exceptions import BadSortFormat, SortFieldNotFound
 
 from db import records
@@ -63,6 +65,66 @@ def test_get_records_orders_before_limiting(roster_table_obj):
     assert record_list[0][7] == 25 and record_list[0][2] == "Amy Gamble"
 
 
+def check_single_field_ordered(record_list, field, direction):
+    for i in range(1, len(record_list)):
+        prev = getattr(record_list[i - 1], field)
+        curr = getattr(record_list[i], field)
+        if prev is None or curr is None:
+            continue
+        comp_func = dir_to_python_func[direction]
+        assert comp_func(prev, curr)
+
+
+def check_multi_field_ordered(record_list, field_dir_pairs):
+    for i in range(1, len(record_list)):
+        for field, direction in field_dir_pairs:
+            prev = getattr(record_list[i - 1], field)
+            curr = getattr(record_list[i], field)
+            if prev is None or curr is None:
+                continue
+
+            comp_func = dir_to_python_func[direction]
+            # If fields are equal, check the next field
+            # If fields differ, ensure the comparison is correct
+            if prev != curr:
+                assert comp_func(prev, curr)
+                break
+
+
+def test_get_records_default_order_single_primary_key(roster_table_obj):
+    roster, engine = roster_table_obj
+    primary_column = roster.primary_key.columns[0].name
+    record_list = records.get_records(roster, engine)
+    check_single_field_ordered(record_list, primary_column, 'asc')
+
+
+def test_get_records_default_order_composite_primary_key(filter_sort_table_obj):
+    filter_sort, engine = filter_sort_table_obj
+    primary_columns = [col.name for col in filter_sort.primary_key.columns]
+    record_list = records.get_records(filter_sort, engine)
+    field_dir_pairs = [(col, 'asc') for col in primary_columns]
+    check_multi_field_ordered(record_list, field_dir_pairs)
+
+
+def test_get_records_default_order_no_primary_key(filter_sort_table_obj):
+    filter_sort, engine = filter_sort_table_obj
+
+    constraint = filter_sort.primary_key
+    with engine.begin() as conn:
+        conn.execute(DropConstraint(constraint))
+    metadata = MetaData(bind=engine)
+    filter_sort = Table(
+        filter_sort.name, metadata, schema=filter_sort.schema, autoload_with=engine
+    )
+    assert len(filter_sort.primary_key.columns) == 0
+
+    record_list = records.get_records(filter_sort, engine)
+
+    columns = [col.name for col in filter_sort.columns]
+    field_dir_pairs = [(col, 'asc') for col in columns]
+    check_multi_field_ordered(record_list, field_dir_pairs)
+
+
 dir_to_python_func = {
     "asc": lambda x, y: x <= y,
     "desc": lambda x, y: x >= y,
@@ -92,13 +154,7 @@ def test_get_records_orders_single_field(
     elif null == "nullslast":
         assert getattr(record_list[-1], field) is None
 
-    for i in range(1, len(record_list)):
-        prev = getattr(record_list[i - 1], field)
-        curr = getattr(record_list[i], field)
-        if prev is None or curr is None:
-            continue
-        comp_func = dir_to_python_func[direction]
-        assert comp_func(prev, curr)
+    check_single_field_ordered(record_list, field, direction)
 
 
 multi_field_test_list = [
@@ -128,18 +184,7 @@ def test_get_records_orders_multiple_fields(
 
     record_list = records.get_records(roster_sort, engine, order_by=order_list)
 
-    for i in range(1, len(record_list)):
-        prev_field_equal = True
-        for field, direction in field_dir_pairs:
-            prev = getattr(record_list[i - 1], field)
-            curr = getattr(record_list[i], field)
-            if prev is None or curr is None:
-                continue
-            comp_func = dir_to_python_func[direction]
-
-            # Only check order when previous field has equal values
-            assert not prev_field_equal or comp_func(prev, curr)
-            prev_field_equal = prev == curr
+    check_multi_field_ordered(record_list, field_dir_pairs)
 
 
 exceptions_test_list = [
diff --git a/db/tests/resources/filter_sort_create.sql b/db/tests/resources/filter_sort_create.sql
--- a/db/tests/resources/filter_sort_create.sql
+++ b/db/tests/resources/filter_sort_create.sql
@@ -1,5 +1,6 @@
 CREATE TABLE "filter_sort" (
     mathesar_id integer NOT NULL,
+    str_id varchar(32) NOT NULL,
     "varchar" varchar(32),
     "numeric" integer,
     "date"    date,
@@ -7,111 +8,111 @@ CREATE TABLE "filter_sort" (
 );
 
 INSERT INTO "filter_sort" VALUES
-(1,'string1',1,'2000-01-01','{}'),
-(2,'string2',2,'2001-01-01','{0,0}'),
-(3,'string3',3,'2002-01-01','{1,1}'),
-(4,'string4',4,'2003-01-01','{2,2}'),
-(5,'string5',5,'2004-01-01','{3,3}'),
-(6,'string6',6,'2005-01-01','{4,4}'),
-(7,'string7',7,'2006-01-01','{5,5}'),
-(8,'string8',8,'2007-01-01','{6,6}'),
-(9,'string9',9,'2008-01-01','{7,7}'),
-(10,'string10',10,'2009-01-01','{8,8}'),
-(11,'string11',11,'2010-01-01','{9,9}'),
-(12,'string12',12,'2011-01-01','{10,10}'),
-(13,'string13',13,'2012-01-01','{11,11}'),
-(14,'string14',14,'2013-01-01','{12,12}'),
-(15,'string15',15,'2014-01-01','{13,13}'),
-(16,'string16',16,'2015-01-01','{14,14}'),
-(17,'string17',17,'2016-01-01','{15,15}'),
-(18,'string18',18,'2017-01-01','{16,16}'),
-(19,'string19',19,'2018-01-01','{17,17}'),
-(20,'string20',20,'2019-01-01','{18,18}'),
-(21,'string21',21,'2020-01-01','{19,19}'),
-(22,'string22',22,'2021-01-01','{20,20}'),
-(23,'string23',23,'2022-01-01','{21,21}'),
-(24,'string24',24,'2023-01-01','{22,22}'),
-(25,'string25',25,'2024-01-01','{23,23}'),
-(26,'string26',26,'2025-01-01','{24,24}'),
-(27,'string27',27,'2026-01-01','{25,25}'),
-(28,'string28',28,'2027-01-01','{26,26}'),
-(29,'string29',29,'2028-01-01','{27,27}'),
-(30,'string30',30,'2029-01-01','{28,28}'),
-(31,'string31',31,'2030-01-01','{29,29}'),
-(32,'string32',32,'2031-01-01','{30,30}'),
-(33,'string33',33,'2032-01-01','{31,31}'),
-(34,'string34',34,'2033-01-01','{32,32}'),
-(35,'string35',35,'2034-01-01','{33,33}'),
-(36,'string36',36,'2035-01-01','{34,34}'),
-(37,'string37',37,'2036-01-01','{35,35}'),
-(38,'string38',38,'2037-01-01','{36,36}'),
-(39,'string39',39,'2038-01-01','{37,37}'),
-(40,'string40',40,'2039-01-01','{38,38}'),
-(41,'string41',41,'2040-01-01','{39,39}'),
-(42,'string42',42,'2041-01-01','{40,40}'),
-(43,'string43',43,'2042-01-01','{41,41}'),
-(44,'string44',44,'2043-01-01','{42,42}'),
-(45,'string45',45,'2044-01-01','{43,43}'),
-(46,'string46',46,'2045-01-01','{44,44}'),
-(47,'string47',47,'2046-01-01','{45,45}'),
-(48,'string48',48,'2047-01-01','{46,46}'),
-(49,'string49',49,'2048-01-01','{47,47}'),
-(50,'string50',50,'2049-01-01','{48,48}'),
-(51,'string51',51,'2050-01-01','{49,49}'),
-(52,'string52',52,'2051-01-01','{50,50}'),
-(53,'string53',53,'2052-01-01','{51,51}'),
-(54,'string54',54,'2053-01-01','{52,52}'),
-(55,'string55',55,'2054-01-01','{53,53}'),
-(56,'string56',56,'2055-01-01','{54,54}'),
-(57,'string57',57,'2056-01-01','{55,55}'),
-(58,'string58',58,'2057-01-01','{56,56}'),
-(59,'string59',59,'2058-01-01','{57,57}'),
-(60,'string60',60,'2059-01-01','{58,58}'),
-(61,'string61',61,'2060-01-01','{59,59}'),
-(62,'string62',62,'2061-01-01','{60,60}'),
-(63,'string63',63,'2062-01-01','{61,61}'),
-(64,'string64',64,'2063-01-01','{62,62}'),
-(65,'string65',65,'2064-01-01','{63,63}'),
-(66,'string66',66,'2065-01-01','{64,64}'),
-(67,'string67',67,'2066-01-01','{65,65}'),
-(68,'string68',68,'2067-01-01','{66,66}'),
-(69,'string69',69,'2068-01-01','{67,67}'),
-(70,'string70',70,'2069-01-01','{68,68}'),
-(71,'string71',71,'2070-01-01','{69,69}'),
-(72,'string72',72,'2071-01-01','{70,70}'),
-(73,'string73',73,'2072-01-01','{71,71}'),
-(74,'string74',74,'2073-01-01','{72,72}'),
-(75,'string75',75,'2074-01-01','{73,73}'),
-(76,'string76',76,'2075-01-01','{74,74}'),
-(77,'string77',77,'2076-01-01','{75,75}'),
-(78,'string78',78,'2077-01-01','{76,76}'),
-(79,'string79',79,'2078-01-01','{77,77}'),
-(80,'string80',80,'2079-01-01','{78,78}'),
-(81,'string81',81,'2080-01-01','{79,79}'),
-(82,'string82',82,'2081-01-01','{80,80}'),
-(83,'string83',83,'2082-01-01','{81,81}'),
-(84,'string84',84,'2083-01-01','{82,82}'),
-(85,'string85',85,'2084-01-01','{83,83}'),
-(86,'string86',86,'2085-01-01','{84,84}'),
-(87,'string87',87,'2086-01-01','{85,85}'),
-(88,'string88',88,'2087-01-01','{86,86}'),
-(89,'string89',89,'2088-01-01','{87,87}'),
-(90,'string90',90,'2089-01-01','{88,88}'),
-(91,'string91',91,'2090-01-01','{89,89}'),
-(92,'string92',92,'2091-01-01','{90,90}'),
-(93,'string93',93,'2092-01-01','{91,91}'),
-(94,'string94',94,'2093-01-01','{92,92}'),
-(95,'string95',95,'2094-01-01','{93,93}'),
-(96,'string96',96,'2095-01-01','{94,94}'),
-(97,'string97',97,'2096-01-01','{95,95}'),
-(98,'string98',98,'2097-01-01','{96,96}'),
-(99,'string99',99,'2098-01-01','{97,97}'),
-(100,'string100',100,'2099-01-01','{98,98}'),
-(101,NULL,NULL,NULL,NULL),
-(102,NULL,NULL,NULL,NULL),
-(103,NULL,NULL,NULL,NULL),
-(104,NULL,NULL,NULL,NULL),
-(105,NULL,NULL,NULL,NULL);
+(1,'a','string1',1,'2000-01-01','{}'),
+(1,'b','string2',2,'2001-01-01','{0,0}'),
+(3,'a','string3',3,'2002-01-01','{1,1}'),
+(3,'b','string4',4,'2003-01-01','{2,2}'),
+(5,'a','string5',5,'2004-01-01','{3,3}'),
+(5,'b','string6',6,'2005-01-01','{4,4}'),
+(7,'a','string7',7,'2006-01-01','{5,5}'),
+(7,'b','string8',8,'2007-01-01','{6,6}'),
+(9,'a','string9',9,'2008-01-01','{7,7}'),
+(9,'b','string10',10,'2009-01-01','{8,8}'),
+(11,'a','string11',11,'2010-01-01','{9,9}'),
+(11,'b','string12',12,'2011-01-01','{10,10}'),
+(13,'a','string13',13,'2012-01-01','{11,11}'),
+(13,'b','string14',14,'2013-01-01','{12,12}'),
+(15,'a','string15',15,'2014-01-01','{13,13}'),
+(15,'b','string16',16,'2015-01-01','{14,14}'),
+(17,'a','string17',17,'2016-01-01','{15,15}'),
+(17,'b','string18',18,'2017-01-01','{16,16}'),
+(19,'a','string19',19,'2018-01-01','{17,17}'),
+(19,'b','string20',20,'2019-01-01','{18,18}'),
+(21,'a','string21',21,'2020-01-01','{19,19}'),
+(21,'b','string22',22,'2021-01-01','{20,20}'),
+(23,'a','string23',23,'2022-01-01','{21,21}'),
+(23,'b','string24',24,'2023-01-01','{22,22}'),
+(25,'a','string25',25,'2024-01-01','{23,23}'),
+(25,'b','string26',26,'2025-01-01','{24,24}'),
+(27,'a','string27',27,'2026-01-01','{25,25}'),
+(27,'b','string28',28,'2027-01-01','{26,26}'),
+(29,'a','string29',29,'2028-01-01','{27,27}'),
+(29,'b','string30',30,'2029-01-01','{28,28}'),
+(31,'a','string31',31,'2030-01-01','{29,29}'),
+(31,'b','string32',32,'2031-01-01','{30,30}'),
+(33,'a','string33',33,'2032-01-01','{31,31}'),
+(33,'b','string34',34,'2033-01-01','{32,32}'),
+(35,'a','string35',35,'2034-01-01','{33,33}'),
+(35,'b','string36',36,'2035-01-01','{34,34}'),
+(37,'a','string37',37,'2036-01-01','{35,35}'),
+(37,'b','string38',38,'2037-01-01','{36,36}'),
+(39,'a','string39',39,'2038-01-01','{37,37}'),
+(39,'b','string40',40,'2039-01-01','{38,38}'),
+(41,'a','string41',41,'2040-01-01','{39,39}'),
+(41,'b','string42',42,'2041-01-01','{40,40}'),
+(43,'a','string43',43,'2042-01-01','{41,41}'),
+(43,'b','string44',44,'2043-01-01','{42,42}'),
+(45,'a','string45',45,'2044-01-01','{43,43}'),
+(45,'b','string46',46,'2045-01-01','{44,44}'),
+(47,'a','string47',47,'2046-01-01','{45,45}'),
+(47,'b','string48',48,'2047-01-01','{46,46}'),
+(49,'a','string49',49,'2048-01-01','{47,47}'),
+(49,'b','string50',50,'2049-01-01','{48,48}'),
+(51,'a','string51',51,'2050-01-01','{49,49}'),
+(51,'b','string52',52,'2051-01-01','{50,50}'),
+(53,'a','string53',53,'2052-01-01','{51,51}'),
+(53,'b','string54',54,'2053-01-01','{52,52}'),
+(55,'a','string55',55,'2054-01-01','{53,53}'),
+(55,'b','string56',56,'2055-01-01','{54,54}'),
+(57,'a','string57',57,'2056-01-01','{55,55}'),
+(57,'b','string58',58,'2057-01-01','{56,56}'),
+(59,'a','string59',59,'2058-01-01','{57,57}'),
+(59,'b','string60',60,'2059-01-01','{58,58}'),
+(61,'a','string61',61,'2060-01-01','{59,59}'),
+(61,'b','string62',62,'2061-01-01','{60,60}'),
+(63,'a','string63',63,'2062-01-01','{61,61}'),
+(63,'b','string64',64,'2063-01-01','{62,62}'),
+(65,'a','string65',65,'2064-01-01','{63,63}'),
+(65,'b','string66',66,'2065-01-01','{64,64}'),
+(67,'a','string67',67,'2066-01-01','{65,65}'),
+(67,'b','string68',68,'2067-01-01','{66,66}'),
+(69,'a','string69',69,'2068-01-01','{67,67}'),
+(69,'b','string70',70,'2069-01-01','{68,68}'),
+(71,'a','string71',71,'2070-01-01','{69,69}'),
+(71,'b','string72',72,'2071-01-01','{70,70}'),
+(73,'a','string73',73,'2072-01-01','{71,71}'),
+(73,'b','string74',74,'2073-01-01','{72,72}'),
+(75,'a','string75',75,'2074-01-01','{73,73}'),
+(75,'b','string76',76,'2075-01-01','{74,74}'),
+(77,'a','string77',77,'2076-01-01','{75,75}'),
+(77,'b','string78',78,'2077-01-01','{76,76}'),
+(79,'a','string79',79,'2078-01-01','{77,77}'),
+(79,'b','string80',80,'2079-01-01','{78,78}'),
+(81,'a','string81',81,'2080-01-01','{79,79}'),
+(81,'b','string82',82,'2081-01-01','{80,80}'),
+(83,'a','string83',83,'2082-01-01','{81,81}'),
+(83,'b','string84',84,'2083-01-01','{82,82}'),
+(85,'a','string85',85,'2084-01-01','{83,83}'),
+(85,'b','string86',86,'2085-01-01','{84,84}'),
+(87,'a','string87',87,'2086-01-01','{85,85}'),
+(87,'b','string88',88,'2087-01-01','{86,86}'),
+(89,'a','string89',89,'2088-01-01','{87,87}'),
+(89,'b','string90',90,'2089-01-01','{88,88}'),
+(91,'a','string91',91,'2090-01-01','{89,89}'),
+(91,'b','string92',92,'2091-01-01','{90,90}'),
+(93,'a','string93',93,'2092-01-01','{91,91}'),
+(93,'b','string94',94,'2093-01-01','{92,92}'),
+(95,'a','string95',95,'2094-01-01','{93,93}'),
+(95,'b','string96',96,'2095-01-01','{94,94}'),
+(97,'a','string97',97,'2096-01-01','{95,95}'),
+(97,'b','string98',98,'2097-01-01','{96,96}'),
+(99,'a','string99',99,'2098-01-01','{97,97}'),
+(99,'b','string100',100,'2099-01-01','{98,98}'),
+(101,'a',NULL,NULL,NULL,NULL),
+(101,'b',NULL,NULL,NULL,NULL),
+(103,'a',NULL,NULL,NULL,NULL),
+(103,'b',NULL,NULL,NULL,NULL),
+(105,'a',NULL,NULL,NULL,NULL);
 
 ALTER TABLE ONLY "filter_sort"
-    ADD CONSTRAINT "filter_sort_pkey" PRIMARY KEY (mathesar_id);
+    ADD CONSTRAINT "filter_sort_pkey" PRIMARY KEY (mathesar_id, str_id);