Repository: timescale/timescaledb (2026-06-02T20:47:06Z)

Issue:
wrong order produced by batch sorted merge in multi-column orderby
```
  CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);                                                                                             
  SELECT create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');                                                                                                
  ALTER TABLE bsm_segby SET (timescaledb.compress,                                                                                                                                   
                             timescaledb.compress_segmentby='grp',                                                                                                                   
                             timescaledb.compress_orderby='name asc, ts asc');                                                                                                       
  INSERT INTO bsm_segby                                                                                                                                                              
  SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4,     
         'k'||lpad(((g*5)%17)::text,3,'0')                                                                                                                                           
  FROM generate_series(0,4379) g;                                                                                                                                                    
  SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;                                                                                                                   
                                                                                                                                                                                     
  -- Expected 0. Returns 1.                                       
  SELECT count(*) misorder FROM (                                                                                                                                                    
    SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt     
    FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s                                                                                                                        
  ) z WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));                                                                                                                     
                                                                                                                                                                                     
  -- Merge-join variant: XX000.                                                                                                                                                      
  SET enable_hashjoin=off; SET enable_nestloop=off;               
  SET max_parallel_workers_per_gather=0;                                                                                                                                             
  SELECT count(*) FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) a                                                                                                          
  JOIN (SELECT name, ts FROM bsm_segby ORDER BY name, ts) b USING (name, ts);                                                                                                        
  -- ERROR: mergejoin input data is out of order                                                                                                                                     
```


Discussion:
Batch Sorted Merge currently relies on minmax batches order which may produce batches overlapping on multikey minmax, and BSM doesn't deal with it correctly. Quickest fix could be to not use BSM on multikey Order By on chunks with no **firstlast** indexes.

The issue does not manifest with  **firstlast** because batches do not overlap with regard to multikey  **firstlast**.

Batches for this example clearly overlap on multikey minmax, i.e. (max_name, max_ts)_1 `(k015, 2024-08-04 00:56:00-04)` is overlapping (min_name, min_ts)_2 `(k015, 2024-08-01 00:44:00-04)` for 2 batches for group 0, but the chunk is considered ordered, i.e. batches are not supposed to overlap. I wonder if there are other issues because of this.

```plpgsql
postgres=# select  grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2, _ts_meta_count  from _timescaledb_internal.compress_hyper_2_2_chunk;
 grp | _ts_meta_min_1 | _ts_meta_max_1 |     _ts_meta_min_2     |     _ts_meta_max_2     | _ts_meta_count
-----+----------------+----------------+------------------------+------------------------+----------------
   0 | k000           | k015           | 2024-08-01 00:00:00-04 | 2024-08-04 00:56:00-04 |           1000
   0 | k015           | k016           | 2024-08-01 00:44:00-04 | 2024-08-04 00:52:00-04 |             95
   1 | k000           | k015           | 2024-08-01 00:01:00-04 | 2024-08-04 00:57:00-04 |           1000
   1 | k015           | k016           | 2024-08-01 01:01:00-04 | 2024-08-04 00:25:00-04 |             95
   2 | k000           | k015           | 2024-08-01 00:02:00-04 | 2024-08-04 00:58:00-04 |           1000
   2 | k015           | k016           | 2024-08-01 00:10:00-04 | 2024-08-04 00:42:00-04 |             95
   3 | k000           | k015           | 2024-08-01 00:03:00-04 | 2024-08-04 00:55:00-04 |           1000
   3 | k015           | k016           | 2024-08-01 00:27:00-04 | 2024-08-04 00:59:00-04 |             95
(8 rows)

``` 



Patch:
diff --git a/.unreleased/pr_9955 b/.unreleased/pr_9955
new file mode 100644
index 00000000000..b3d16b495bd
--- /dev/null
+++ b/.unreleased/pr_9955
@@ -0,0 +1,1 @@
+Fixes: #9955 Fix wrong results when using Batch Sorted Merge with no first-last index on a non-leading order by column
diff --git a/tsl/src/nodes/columnar_scan/columnar_scan.c b/tsl/src/nodes/columnar_scan/columnar_scan.c
index be0a8a0544e..aba6171c914 100644
--- a/tsl/src/nodes/columnar_scan/columnar_scan.c
+++ b/tsl/src/nodes/columnar_scan/columnar_scan.c
@@ -2925,15 +2925,33 @@ match_pathkeys_to_compression_orderby(List *pathkeys, List *chunk_em_exprs,
 			return false;
 		}
 
-		/* Bail out on BSM if orderby column is nullable,
-		 * as at the moment the minmax metadata we have doesn't include NULLs,
-		 * so it's difficult to use it for null-sensitive ordering.
-		 * But this restriction can be lifted in the future on new type of chunks
-		 * with NULL-handling metadata.
-		 */
-		if (for_bsm && !is_var_notnull(compression_info, var))
+		if (for_bsm)
 		{
-			return false;
+			/* Bail out on Batch Sorted Merge if orderby column is nullable,
+			 * as at the moment the minmax metadata we have doesn't include NULLs,
+			 * so it's difficult to use it for null-sensitive ordering.
+			 * But this restriction can be lifted in the future on new type of chunks
+			 * with NULL-handling metadata.
+			 */
+			if (!is_var_notnull(compression_info, var))
+			{
+				return false;
+			}
+
+			/* Bail out on Batch Sorted Merge with multiple order by keys
+			 * if non-leading keys don't use firstlast index.
+			 * Batches can be sorted incorrectly on multikey minmax index,
+			 * for example
+			 * [(1, 20) .. (1, 30), (2,0)...(2,30)] with min(1),(0)
+			 * will be sorted before  [(1,1) ..  (1,19)] with min(1),(1)
+			 * but it should be sorted after as (1,20) > (1,1): correct with firstlast index.
+			 */
+			if (compressed_pk_index > 1 &&
+				orderby_sparse_kind(compression_info->settings, orderby_index) !=
+					ORDERBY_SPARSE_FIRSTLAST)
+			{
+				return false;
+			}
 		}
 
 		bool orderby_desc =


Test patch:
diff --git a/tsl/test/expected/compression_sorted_merge.out b/tsl/test/expected/compression_sorted_merge.out
index c62c9d9a49e..8a095c46d5b 100644
--- a/tsl/test/expected/compression_sorted_merge.out
+++ b/tsl/test/expected/compression_sorted_merge.out
@@ -1365,6 +1365,7 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign
 --------------------------+---------+----------+---------+--------------+-------------+----------------------+---------------------
  Wed Jan 01 10:00:00 2020 | hin1111 | model111 | blok111 | message_here | signal1     |                12.34 | 12.34
 
+drop table bugtab cascade;
 -- Condition that filter the first tuple of a batch - Issue 5797
 CREATE TABLE test (
     id      bigint,
@@ -1528,4 +1529,87 @@ SELECT t.dttm FROM test t ORDER BY t.dttm LIMIT 1;
                      Output: compress_hyper_14_26_chunk._ts_meta_count, compress_hyper_14_26_chunk.otherid, compress_hyper_14_26_chunk.valuefk, compress_hyper_14_26_chunk.otherfk, compress_hyper_14_26_chunk.id, compress_hyper_14_26_chunk._ts_meta_min_1, compress_hyper_14_26_chunk._ts_meta_max_1, compress_hyper_14_26_chunk._ts_meta_v2_first_dttm, compress_hyper_14_26_chunk._ts_meta_v2_last_dttm, compress_hyper_14_26_chunk.dttm, compress_hyper_14_26_chunk.measure
 
 RESET enable_sort;
+drop table test cascade;
+-- Test issue #9922: wrong sort order for Batch Sorted Merge with multikey minmax index
+\c :TEST_DBNAME :ROLE_SUPERUSER
+CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);
+SELECT table_name FROM create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');
+ table_name 
+------------
+ bsm_segby
+
+ALTER TABLE bsm_segby SET (timescaledb.compress,  timescaledb.compress_segmentby='grp', timescaledb.compress_orderby='name asc, ts asc');
+INSERT INTO bsm_segby
+SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4, 'k'||lpad(((g*5)%17)::text,3,'0')
+FROM generate_series(0,4379) g;
+SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;
+ count 
+-------
+     1
+
+select schema_name || '.' || table_name chunk  from _timescaledb_catalog.chunk
+    where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
+        where hypertable_id = (select id from _timescaledb_catalog.hypertable
+            where table_name = 'bsm_segby') limit 1)
+\gset
+-- Firstlast index is used with Batch Sorted Merge: correct result
+set timescaledb.debug_require_batch_sorted_merge = 'force';
+SELECT count(*) misorder FROM (
+  SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
+  FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
+WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
+ misorder 
+----------
+        0
+
+-- Remove firstlast index from leading column: still OK to use Batch Sorted Merge
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
+where relid = 'bsm_segby'::regclass;
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
+where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
+    where id = (select compressed_chunk_id  from _timescaledb_catalog.chunk
+        where hypertable_id = (select id from _timescaledb_catalog.hypertable
+            where table_name = 'bsm_segby') limit 1));
+create index compressed_index_minmax_firstlast on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_v2_first_ts, _ts_meta_v2_last_ts);
+-- Correct result with Batch Sorted Merge
+set timescaledb.debug_require_batch_sorted_merge = 'force';
+SELECT count(*) misorder FROM (
+  SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
+  FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
+WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
+ misorder 
+----------
+        0
+
+-- Now use minmax index on non-leading column "ts"
+drop index _timescaledb_internal.compressed_index_minmax_firstlast;
+create index compressed_index_minmax_minmax on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2);
+-- Remove firstlast index from all columns
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
+where relid = 'bsm_segby'::regclass;
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
+where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
+    where id = (select compressed_chunk_id  from _timescaledb_catalog.chunk
+        where hypertable_id = (select id from _timescaledb_catalog.hypertable
+            where table_name = 'bsm_segby') limit 1));
+-- Correct result only when cannot use Batch Sorted Merge
+set timescaledb.debug_require_batch_sorted_merge = 'forbid';
+SELECT count(*) misorder FROM (
+  SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
+  FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
+WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
+ misorder 
+----------
+        0
+
+drop table bsm_segby cascade;
+drop table test1 cascade;
+drop table test2 cascade;
+drop table test_with_defined_null cascade;
+drop table test_costs cascade;
+drop table insert_test cascade;
 reset timescaledb.debug_require_batch_sorted_merge;
diff --git a/tsl/test/sql/compression_sorted_merge.sql b/tsl/test/sql/compression_sorted_merge.sql
index cbd41e6f48b..0cd7ab9b55b 100644
--- a/tsl/test/sql/compression_sorted_merge.sql
+++ b/tsl/test/sql/compression_sorted_merge.sql
@@ -555,6 +555,8 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign
 
 SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"signal_name"::text,"signal_numeric_value","signal_string_value"::text FROM bugtab ORDER BY "time" DESC;
 
+drop table bugtab cascade;
+
 -- Condition that filter the first tuple of a batch - Issue 5797
 CREATE TABLE test (
     id      bigint,
@@ -666,5 +668,88 @@ set timescaledb.debug_require_batch_sorted_merge = 'require';
 SELECT t.dttm FROM test t ORDER BY t.dttm LIMIT 1;
 RESET enable_sort;
 
+drop table test cascade;
+
+-- Test issue #9922: wrong sort order for Batch Sorted Merge with multikey minmax index
+\c :TEST_DBNAME :ROLE_SUPERUSER
+
+CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);
+SELECT table_name FROM create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');
+ALTER TABLE bsm_segby SET (timescaledb.compress,  timescaledb.compress_segmentby='grp', timescaledb.compress_orderby='name asc, ts asc');
+
+INSERT INTO bsm_segby
+SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4, 'k'||lpad(((g*5)%17)::text,3,'0')
+FROM generate_series(0,4379) g;
+
+SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;
+
+select schema_name || '.' || table_name chunk  from _timescaledb_catalog.chunk
+    where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
+        where hypertable_id = (select id from _timescaledb_catalog.hypertable
+            where table_name = 'bsm_segby') limit 1)
+\gset
+
+-- Firstlast index is used with Batch Sorted Merge: correct result
+set timescaledb.debug_require_batch_sorted_merge = 'force';
+
+SELECT count(*) misorder FROM (
+  SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
+  FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
+WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
+
+-- Remove firstlast index from leading column: still OK to use Batch Sorted Merge
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
+where relid = 'bsm_segby'::regclass;
+
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
+where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
+    where id = (select compressed_chunk_id  from _timescaledb_catalog.chunk
+        where hypertable_id = (select id from _timescaledb_catalog.hypertable
+            where table_name = 'bsm_segby') limit 1));
+
+create index compressed_index_minmax_firstlast on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_v2_first_ts, _ts_meta_v2_last_ts);
+
+-- Correct result with Batch Sorted Merge
+set timescaledb.debug_require_batch_sorted_merge = 'force';
+
+SELECT count(*) misorder FROM (
+  SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
+  FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
+WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
+
+-- Now use minmax index on non-leading column "ts"
+drop index _timescaledb_internal.compressed_index_minmax_firstlast;
+create index compressed_index_minmax_minmax on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2);
+
+-- Remove firstlast index from all columns
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
+where relid = 'bsm_segby'::regclass;
+
+update _timescaledb_catalog.compression_settings
+set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
+where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
+    where id = (select compressed_chunk_id  from _timescaledb_catalog.chunk
+        where hypertable_id = (select id from _timescaledb_catalog.hypertable
+            where table_name = 'bsm_segby') limit 1));
+
+-- Correct result only when cannot use Batch Sorted Merge
+set timescaledb.debug_require_batch_sorted_merge = 'forbid';
+
+SELECT count(*) misorder FROM (
+  SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
+  FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
+WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
+
+drop table bsm_segby cascade;
+
+drop table test1 cascade;
+drop table test2 cascade;
+drop table test_with_defined_null cascade;
+drop table test_costs cascade;
+drop table insert_test cascade;
+
 reset timescaledb.debug_require_batch_sorted_merge;