Repository: apache/cloudberry
Pull request #1010: Optimize AOCS scan performance by introducing specialized no-qual path 
Author: login: yjhjstz
Labels: name: type: Performance
description: cloudberry runs slow on some particular query
Created: 2025-03-26T17:03:13Z | Merged: 2025-03-27T14:19:55Z

This patch improves the performance of Append-Only Columnar Store (AOCS) table
scans by introducing `aocs_getnext_noqual`, a specialized scan function optimized
for cases where no scan qualifiers are present. This change helps avoid unnecessary
checks, reducing CPU overhead during sequential scans.

Key changes:
- Added `aocs_getnext_noqual`, which eliminates redundant checks in scans.
- Introduced `pg_attribute_hot_inline` for `aocs_getnext_noqual` to improve inlining efficiency.
- Refactored related scan functions to integrate the optimized path while maintaining correctness.
- Avoiding unnecessary setup when predicate pushdown is not applicable.

These changes are expected to improve scan efficiency for AOCS tables,
especially for full-table sequential scans.

Author: Jianghua Yang
Fixes #ISSUE_Number

### What does this PR do?
<!-- Brief overview of the changes, including any major features or fixes -->

### Type of Change
- [ ] Bug fix (non-breaking change)
- [ ] New feature (non-breaking change)
- [ ] Breaking change (fix or feature with breaking changes)
- [ ] Documentation update

### Breaking Changes
<!-- Remove if not applicable. If yes, explain impact and migration path -->

### Test Plan
<!-- How did you test these changes? -->
- [ ] Unit tests added/updated
- [ ] Integration tests added/updated
- [ ] Passed `make installcheck`
- [ ] Passed `make -C src/test installcheck-cbdb-parallel`

### Impact
<!-- Remove sections that don't apply -->
**Performance:**
<!-- Any performance implications? -->

**User-facing changes:**
<!-- Any changes visible to users? -->

**Dependencies:**
<!-- New dependencies or version changes? -->

### Checklist
- [ ] Followed [contribution guide](https://cloudberry.apache.org/contribute/code)
- [ ] Added/updated documentation
- [ ] Reviewed code for security implications
- [ ] Requested review from [cloudberry committers](https://github.com/orgs/apache/teams/cloudberry-committers)

### Additional Context
<!-- Any other information that would help reviewers? Remove if none -->

### CI Skip Instructions
<!--
To skip CI builds, add the appropriate CI skip identifier to your PR title.
The identifier must:
- Be in square brackets []
- Include the word "ci" and either "skip" or "no"
- Only use for documentation-only changes or when absolutely necessary
-->

---
<!-- Join our community:
- Mailing list: [dev@cloudberry.apache.org](https://lists.apache.org/list.html?dev@cloudberry.apache.org) (subscribe: dev-subscribe@cloudberry.apache.org)
- Discussions: https://github.com/apache/cloudberry/discussions -->



Diff:
diff --git a/src/backend/access/aocs/aocsam.c b/src/backend/access/aocs/aocsam.c
index 978252d526c..be0c07e9bfa 100644
--- a/src/backend/access/aocs/aocsam.c
+++ b/src/backend/access/aocs/aocsam.c
@@ -675,82 +675,140 @@ aocs_endscan(AOCSScanDesc scan)
 	pfree(scan);
 }
 
-/*
- * Upgrades a Datum value from a previous version of the AOCS page format. The
- * DatumStreamRead that is passed must correspond to the column being upgraded.
- */
-static void upgrade_datum_impl(DatumStreamRead *ds, int attno, Datum values[],
-							   bool isnull[], int formatversion)
+
+static pg_attribute_hot_inline bool
+aocs_getnext_noqual(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
-	bool 	convert_numeric = false;
+	Datum	   *d = slot->tts_values;
+	bool	   *null = slot->tts_isnull;
+	AOTupleId	aoTupleId;
+	int64		rowNum = INT64CONST(-1);
+	int			err = 0;
+	bool		isSnapshotAny = (scan->rs_base.rs_snapshot == SnapshotAny);
+	AttrNumber	natts;
 
-	if (PG82NumericConversionNeeded(formatversion))
+	Assert(ScanDirectionIsForward(direction));
+
+	if (scan->columnScanInfo.relationTupleDesc == NULL)
 	{
-		/*
-		 * On the first call for this DatumStream, figure out if this column is
-		 * a numeric, or a domain over numerics.
-		 *
-		 * TODO: consolidate this code with upgrade_tuple() in appendonlyam.c.
-		 */
-		if (!OidIsValid(ds->baseTypeOid))
+		scan->columnScanInfo.relationTupleDesc = slot->tts_tupleDescriptor;
+		/* Pin it! ... and of course release it upon destruction / rescan */
+		PinTupleDesc(scan->columnScanInfo.relationTupleDesc);
+		initscan_with_colinfo(scan);
+	}
+
+	natts = slot->tts_tupleDescriptor->natts;
+	Assert(natts <= scan->columnScanInfo.relationTupleDesc->natts);
+
+	while (1)
+	{
+		AOCSFileSegInfo *curseginfo;
+		bool visible_pass;
+
+ReadNext:
+		/* If necessary, open next seg */
+		if (scan->cur_seg < 0 || err < 0)
 		{
-			ds->baseTypeOid = getBaseType(ds->typeInfo.typid);
+			err = open_next_scan_seg(scan);
+			if (err < 0)
+			{
+				/* No more seg, we are at the end */
+				ExecClearTuple(slot);
+				scan->cur_seg = -1;
+				return false;
+			}
+			scan->segrowsprocessed = 0;
 		}
 
-		/* If this Datum is a numeric, we need to convert it. */
-		convert_numeric = (ds->baseTypeOid == NUMERICOID) && !isnull[attno];
-	}
+		Assert(scan->cur_seg >= 0);
+		curseginfo = scan->seginfo[scan->cur_seg];
 
-	if (convert_numeric)
-	{
-		/*
-		 * Before PostgreSQL 8.3, the n_weight and n_sign_dscale fields were the
-		 * other way 'round. Swap them.
-		 */
-		Datum 		datum;
-		char	   *numericdata;
-		char	   *upgradedata;
-		size_t		datalen;
-		uint16		tmp;
+		/* Read from cur_seg */
+		visible_pass = true;
+		for (AttrNumber i = 0; i < scan->columnScanInfo.num_proj_atts; i++)
+		{
+			AttrNumber	attno = scan->columnScanInfo.proj_atts[i];
 
-		/*
-		 * We need to make a copy of this data so that any other tuples pointing
-		 * to it won't be affected. Store it in the upgrade space for this
-		 * DatumStream.
-		 */
-		datum = values[attno];
-		datalen = VARSIZE_ANY(DatumGetPointer(datum));
+			err = datumstreamread_advance(scan->columnScanInfo.ds[attno]);
+			Assert(err >= 0);
+			if (err == 0)
+			{
+				err = datumstreamread_block(scan->columnScanInfo.ds[attno], scan->blockDirectory, attno);
+				if (err < 0)
+				{
+					/*
+					 * Ha, cannot read next block, we need to go to next seg
+					 */
+					close_cur_scan_seg(scan);
+					goto ReadNext;
+				}
 
-		upgradedata = datumstreamread_get_upgrade_space(ds, datalen);
-		memcpy(upgradedata, DatumGetPointer(datum), datalen);
+				AOCSScanDesc_UpdateTotalBytesRead(scan, attno);
 
-		/* Swap the fields. */
-		numericdata = VARDATA_ANY(upgradedata);
+				err = datumstreamread_advance(scan->columnScanInfo.ds[attno]);
+				Assert(err > 0);
+			}
+			if (!visible_pass)
+				continue; /* not break, need advance for other cols */
 
-		memcpy(&tmp, &numericdata[0], 2);
-		memcpy(&numericdata[0], &numericdata[2], 2);
-		memcpy(&numericdata[2], &tmp, 2);
+			/*
+			 * Get the column's datum right here since the data structures
+			 * should still be hot in CPU data cache memory.
+			 */
+			datumstreamread_get(scan->columnScanInfo.ds[attno], &d[attno], &null[attno]);
 
-		/* Re-point the Datum to the upgraded numeric. */
-		values[attno] = PointerGetDatum(upgradedata);
-	}
-}
+			if (i == 0)
+			{
+				if (rowNum == INT64CONST(-1) &&
+					scan->columnScanInfo.ds[attno]->blockFirstRowNum != INT64CONST(-1))
+				{
+					Assert(scan->columnScanInfo.ds[attno]->blockFirstRowNum > 0);
+					rowNum = scan->columnScanInfo.ds[attno]->blockFirstRowNum +
+						datumstreamread_nth(scan->columnScanInfo.ds[attno]);
+				}
+				scan->segrowsprocessed++;
+				if (rowNum == INT64CONST(-1))
+				{
+					AOTupleIdInit(&aoTupleId, curseginfo->segno, scan->segrowsprocessed);
+				}
+				else
+				{
+					AOTupleIdInit(&aoTupleId, curseginfo->segno, rowNum);
+				}
 
-static void upgrade_datum_scan(AOCSScanDesc scan, int attno, Datum values[],
-							   bool isnull[], int formatversion)
-{
-	upgrade_datum_impl(scan->columnScanInfo.ds[attno], attno, values, isnull, formatversion);
-}
+				if (!isSnapshotAny && !AppendOnlyVisimap_IsVisible(&scan->visibilityMap, &aoTupleId))
+				{
+					/*
+					 * The tuple is invisible.
+					 * In `analyze`, we can simply return false
+					 */
+					if ((scan->rs_base.rs_flags & SO_TYPE_ANALYZE) != 0)
+						return false;
 
-static void upgrade_datum_fetch(AOCSFetchDesc fetch, int attno, Datum values[],
-								bool isnull[], int formatversion)
-{
-	upgrade_datum_impl(fetch->datumStreamFetchDesc[attno]->datumStream, attno,
-					   values, isnull, formatversion);
+					rowNum = INT64CONST(-1);
+					visible_pass = false;
+					continue; /* not break, need advance for other cols */
+				}
+			}
+		}
+		if (!visible_pass)
+		{
+			rowNum = INT64CONST(-1);
+			goto ReadNext;
+		}
+		scan->cdb_fake_ctid = *((ItemPointer) &aoTupleId);
+
+		slot->tts_nvalid = natts;
+		slot->tts_tid = scan->cdb_fake_ctid;
+		return true;
+	}
+
+	Assert(!"Never here");
+	return false;
 }
 
-bool
-aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
+static bool
+aocs_getnext_withqual(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
 	if (scan->aos_pushdown_qual && scan->aos_scaned_rows < scan->aos_sample_rows)
 	{
@@ -849,15 +907,6 @@ aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 			 */
 			datumstreamread_get(scan->columnScanInfo.ds[attno], &d[attno], &null[attno]);
 
-			/*
-			 * Perform any required upgrades on the Datum we just fetched.
-			 */
-			if (curseginfo->formatversion < AOSegfileFormatVersion_GetLatest ())
-			{
-				upgrade_datum_scan(scan, attno, d, null,
-								   curseginfo->formatversion);
-			}
-
 			if (i == 0)
 			{
 				if (rowNum == INT64CONST(-1) &&
@@ -911,6 +960,21 @@ aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 }
 
 
+pg_attribute_hot bool
+aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
+{
+	/*
+	* [0] = noqual, [1] = withqual
+	*/
+	static bool (* const getnext_impl[2])(AOCSScanDesc, ScanDirection, TupleTableSlot*) = {
+		aocs_getnext_noqual,
+		aocs_getnext_withqual
+	};
+
+	return getnext_impl[!!scan->aos_pushdown_qual](scan, direction, slot);
+}
+
+
 /* Open next file segment for write.  See SetCurrentFileSegForWrite */
 /* XXX Right now, we put each column to different files */
 static void
@@ -1291,18 +1355,9 @@ fetchFromCurrentBlock(AOCSFetchDesc aocsFetchDesc,
 	{
 		Datum	   *values = slot->tts_values;
 		bool	   *nulls = slot->tts_isnull;
-		int			formatversion = datumStream->ao_read.formatVersion;
 
 		datumstreamread_get(datumStream, &(values[colno]), &(nulls[colno]));
 
-		/*
-		 * Perform any required upgrades on the Datum we just fetched.
-		 */
-		if (formatversion < AOSegfileFormatVersion_GetLatest ())
-		{
-			upgrade_datum_fetch(aocsFetchDesc, colno, values, nulls,
-								formatversion);
-		}
 	}
 	else
 	{
@@ -2364,6 +2419,8 @@ aocs_predicate_pushdown_prepare(AOCSScanDesc scan,
 								ExprContext *ecxt,
 								PlanState *ps)
 {
+	if (!qual)
+		return state;
 	int  ncol  = scan->rs_base.rs_rd->rd_att->natts;
 
 	List **qual_list = (List **)palloc0(sizeof(List *) * ncol);
@@ -2375,8 +2432,6 @@ aocs_predicate_pushdown_prepare(AOCSScanDesc scan,
 	scan->aos_scaned_rows       = 0;
 	scan->aos_qual_rows         = (int *)palloc0(sizeof(int) * ncol);
 
-	if (!qual)
-		return state;
 	bool *proj = palloc0(ncol * sizeof(bool));
 	int num_qual_atts = 0;
 	int *qual_atts    = palloc(ncol * sizeof(int));
@@ -2598,15 +2653,6 @@ aocs_getnext_sample(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *
 			 */
 			datumstreamread_get(scan->columnScanInfo.ds[attno], &d[attno], &null[attno]);
 
-			/*
-			 * Perform any required upgrades on the Datum we just fetched.
-			 */
-			if (curseginfo->formatversion < AOSegfileFormatVersion_GetLatest ())
-			{
-				upgrade_datum_scan(scan, attno, d, null,
-								   curseginfo->formatversion);
-			}
-
 			/*
 			 * set rowNum, aoTupleId and test visibility.
 			 */
diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c
index d213a23fc9c..fda36094cf3 100755
--- a/src/backend/access/appendonly/appendonlyam.c
+++ b/src/backend/access/appendonly/appendonlyam.c
@@ -783,141 +783,6 @@ AppendOnlyExecutorReadBlock_ResetCounts(AppendOnlyExecutorReadBlock *executorRea
 	executorReadBlock->totalRowsScanned = 0;
 }
 
-/*
- * Given a tuple in 'formatversion', convert it to a format that is
- * understood by the rest of the system.
- */
-static MemTuple
-upgrade_tuple(AppendOnlyExecutorReadBlock *executorReadBlock,
-			  MemTuple mtup, MemTupleBinding *pbind, int formatversion, bool *shouldFree)
-{
-	TupleDesc	tupdesc = pbind->tupdesc;
-	const int	natts = tupdesc->natts;
-	MemTuple	newtuple;
-	int			i;
-
-	static Datum *values = NULL;
-	static bool *isnull = NULL;
-	static int	nallocated = 0;
-
-	bool		convert_alignment = false;
-	bool		convert_numerics = false;
-
-	/*
-	 * MPP-7372: If the AO table was created before the fix for this issue, it
-	 * may contain tuples with misaligned bindings. Here we check if the
-	 * stored memtuple is problematic and then create a clone of the tuple
-	 * with properly aligned bindings to be used by the executor.
-	 */
-	if (formatversion < AOSegfileFormatVersion_Aligned64bit &&
-		memtuple_has_misaligned_attribute(mtup, pbind))
-		convert_alignment = true;
-
-	if (PG82NumericConversionNeeded(formatversion))
-	{
-		/*
-		 * On first call, figure out which columns are numerics, or domains
-		 * over numerics.
-		 */
-		if (executorReadBlock->numericAtts == NULL)
-		{
-			int			n;
-
-			executorReadBlock->numericAtts = (int *) palloc(natts * sizeof(int));
-
-			n = 0;
-			for (i = 0; i < natts; i++)
-			{
-				Oid			typeoid;
-
-				typeoid = getBaseType(TupleDescAttr(tupdesc, i)->atttypid);
-				if (typeoid == NUMERICOID)
-					executorReadBlock->numericAtts[n++] = i;
-			}
-			executorReadBlock->numNumericAtts = n;
-		}
-
-		/* If there were any numeric columns, we need to convert them. */
-		if (executorReadBlock->numNumericAtts > 0)
-			convert_numerics = true;
-	}
-
-	if (!convert_alignment && !convert_numerics)
-	{
-		/* No conversion required. Return the original tuple unmodified. */
-		*shouldFree = false;
-		return mtup;
-	}
-
-	/* Conversion is needed. */
-
-	/* enlarge the arrays if needed */
-	if (natts > nallocated)
-	{
-		if (values)
-			pfree(values);
-		if (isnull)
-			pfree(isnull);
-		values = (Datum *) MemoryContextAlloc(TopMemoryContext, natts * sizeof(Datum));
-		isnull = (bool *) MemoryContextAlloc(TopMemoryContext, natts * sizeof(bool));
-		nallocated = natts;
-	}
-
-	if (convert_alignment)
-	{
-		/* get attribute values form mis-aligned tuple */
-		memtuple_deform_misaligned(mtup, pbind, values, isnull);
-		/* Form a new, properly-aligned, tuple */
-		newtuple = memtuple_form(pbind, values, isnull);
-	}
-	else
-	{
-		/*
-		 * make a modifiable copy
-		 */
-		newtuple = memtuple_copy(mtup);
-	}
-
-	/*
-	 * NOTE: we do this *after* creating the new tuple, so that we can modify
-	 * the new, copied, tuple in-place.
-	 */
-	if (convert_numerics)
-	{
-		int			i;
-
-		/*
-		 * Get pointers to the datums within the tuple
-		 */
-		memtuple_deform(newtuple, pbind, values, isnull);
-
-		for (i = 0; i < executorReadBlock->numNumericAtts; i++)
-		{
-			/*
-			 * Before PostgreSQL 8.3, the n_weight and n_sign_dscale fields
-			 * were the other way 'round. Swap them.
-			 */
-			Datum		datum;
-			char	   *numericdata;
-			uint16		tmp;
-
-			if (isnull[executorReadBlock->numericAtts[i]])
-				continue;
-
-			datum = values[executorReadBlock->numericAtts[i]];
-			numericdata = VARDATA_ANY(DatumGetPointer(datum));
-
-			memcpy(&tmp, &numericdata[0], 2);
-			memcpy(&numericdata[0], &numericdata[2], 2);
-			memcpy(&numericdata[2], &tmp, 2);
-		}
-	}
-
-	*shouldFree = true;
-	return newtuple;
-}
-
-
 static void
 AOExecutorReadBlockBindingInit(AppendOnlyExecutorReadBlock *executorReadBlock,
 									   TupleTableSlot *slot)
@@ -970,11 +835,6 @@ AppendOnlyExecutorReadBlock_ProcessTuple(AppendOnlyExecutorReadBlock *executorRe
 
 		Assert(executorReadBlock->mt_bind);
 
-		/* If the tuple is not in the latest format, convert it */
-		// GPDB_12_MERGE_FIXME: Is pg_upgrade from old versions still a thing? Can we drop this?
-		if (formatVersion < AOSegfileFormatVersion_GetLatest ())
-			tuple = upgrade_tuple(executorReadBlock, tuple, executorReadBlock->mt_bind, formatVersion, &shouldFree);
-
 		ExecClearTuple(slot);
 		memtuple_deform(tuple, executorReadBlock->mt_bind, slot->tts_values, slot->tts_isnull);
 		slot->tts_tid = fake_ctid;
diff --git a/src/include/c.h b/src/include/c.h
index 3b83ea6ca70..a90e823999d 100644
--- a/src/include/c.h
+++ b/src/include/c.h
@@ -181,6 +181,38 @@
 #define pg_attribute_noreturn()
 #endif
 
+/*
+ * Cross-platform macro for hot+inline function attributes
+ * Combines HOT path optimization with forced inlining
+ */
+#if (defined(__GNUC__) && __GNUC__ >= 4 && defined(__OPTIMIZE__)) || defined(__clang__)
+/*
+ * Native support in GCC >= 4 and Clang:
+ * - always_inline: Force inlining regardless of optimization level
+ * - hot: Mark as frequently executed function (prioritize optimization)
+ */
+#define pg_attribute_hot_inline __attribute__((hot, always_inline)) inline
+#elif defined(_MSC_VER)
+/*
+ * MSVC equivalent:
+ * - __forceinline: Similar to always_inline
+ * Note: MSVC lacks direct HOT attribute equivalent
+ */
+#define pg_attribute_hot_inline __forceinline
+#elif defined(__SUNPRO_C) || defined(__IBMC__)
+/*
+ * SunPro and IBM XL C compilers:
+ * Only partial support - gets always_inline but not hot marking
+ */
+#define pg_attribute_hot_inline __attribute__((always_inline)) inline
+#else
+/*
+ * Universal fallback for other compilers:
+ * Basic inlining without optimization hints
+ */
+#define pg_attribute_hot_inline inline
+#endif
+
 /*
  * Use "pg_attribute_always_inline" in place of "inline" for functions that
  * we wish to force inlining of, even when the compiler's heuristics would
diff --git a/src/include/catalog/pg_appendonly.h b/src/include/catalog/pg_appendonly.h
index 651f915da49..d73b0661990 100644
--- a/src/include/catalog/pg_appendonly.h
+++ b/src/include/catalog/pg_appendonly.h
@@ -118,15 +118,6 @@ static inline void AOSegfileFormatVersion_CheckValid(int version)
 	(version > AOSegfileFormatVersion_Original) \
 )
 
-/*
- * Are numerics stored in old, pre-PostgreSQL 8.3 format, and need converting?
- */
-#define PG82NumericConversionNeeded(version) \
-( \
-	AOSegfileFormatVersion_CheckValid(version), \
-	(version > AOSegfileFormatVersion_Original) \
-)
-
 extern void
 InsertAppendOnlyEntry(Oid relid,
 					  int blocksize,
diff --git a/src/include/utils/datumstream.h b/src/include/utils/datumstream.h
index 20e0e5b2134..5210159d8b6 100644
--- a/src/include/utils/datumstream.h
+++ b/src/include/utils/datumstream.h
@@ -209,10 +209,10 @@ datumstreamread_get(DatumStreamRead * acc, Datum *datum, bool *null)
 }
 
 extern int	datumstreamread_advancelarge(DatumStreamRead * ds);
-inline static int
+static pg_attribute_always_inline int
 datumstreamread_advance(DatumStreamRead * acc)
 {
-	if (acc->largeObjectState == DatumStreamLargeObjectState_None)
+	if (unlikely(acc->largeObjectState == DatumStreamLargeObjectState_None))
 	{
 		/*
 		 * Small objects are handled by the DatumStreamBlockRead module.
diff --git a/src/include/utils/datumstreamblock.h b/src/include/utils/datumstreamblock.h
index 24390745d1b..43b8d8eda90 100755
--- a/src/include/utils/datumstreamblock.h
+++ b/src/include/utils/datumstreamblock.h
@@ -1438,7 +1438,7 @@ DatumStreamBlockRead_Get(DatumStreamBlockRead * dsr, Datum *datum, bool *null)
 	}
 }
 
-inline static int
+static pg_attribute_always_inline int
 DatumStreamBlockRead_AdvanceOrig(DatumStreamBlockRead * dsr)
 {
 	Assert(dsr);
@@ -1721,7 +1721,7 @@ DatumStreamBlockRead_AdvanceDenseDelta(DatumStreamBlockRead * dsr)
 	return DELTA_COMPRESSION_OK;
 }
 
-inline static int
+static pg_attribute_always_inline int
 DatumStreamBlockRead_AdvanceDense(DatumStreamBlockRead * dsr)
 {
 	Assert(dsr);
@@ -2035,7 +2035,7 @@ DatumStreamBlockRead_AdvanceDense(DatumStreamBlockRead * dsr)
 	return 1;
 }
 
-inline static int
+static pg_attribute_always_inline int
 DatumStreamBlockRead_Advance(DatumStreamBlockRead * dsr)
 {
 	/*
diff --git a/src/test/isolation2/.gitignore b/src/test/isolation2/.gitignore
index c95d47213bc..3e18c982138 100644
--- a/src/test/isolation2/.gitignore
+++ b/src/test/isolation2/.gitignore
@@ -2,9 +2,6 @@ regression.diffs
 regression.out
 */dummy.sql
 */dummy.out
-sql/ao_upgrade.sql
-expected/ao_upgrade.out
-expected/ao_upgrade_optimizer.out
 sql/external_table.sql
 expected/external_table.out
 sql/fts_manual_probe.sql
diff --git a/src/test/isolation2/expected/.gitignore b/src/test/isolation2/expected/.gitignore
index dca717117a4..610e9b171dd 100644
--- a/src/test/isolation2/expected/.gitignore
+++ b/src/test/isolation2/expected/.gitignore
@@ -1,6 +1,4 @@
 # ignores only for this directory
-/ao_upgrade.out
-/ao_upgrade_optimizer.out
 /external_table.out
 /fts_manual_probe.out
 /gp_collation.out
diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule
index f1ac29f0c5b..18fd4a2c0c4 100644
--- a/src/test/isolation2/isolation2_schedule
+++ b/src/test/isolation2/isolation2_schedule
@@ -79,7 +79,6 @@ test: distributedlog-bug
 test: invalidated_toast_index
 test: distributed_snapshot
 test: gp_collation
-test: ao_upgrade
 test: ao_blkdir
 test: bitmap_index_concurrent
 test: bitmap_index_crash
diff --git a/src/test/isolation2/sql/.gitignore b/src/test/isolation2/sql/.gitignore
index 28a68f8baa6..361b986e18d 100644
--- a/src/test/isolation2/sql/.gitignore
+++ b/src/test/isolation2/sql/.gitignore
@@ -1,5 +1,4 @@
 # ignores only for this directory
-/ao_upgrade.sql
 /external_table.sql
 /fts_manual_probe.sql
 /gp_collation.sql
diff --git a/src/test/regress/sql/appendonly_format.sql b/src/test/regress/sql/appendonly_format.sql
deleted file mode 100644
index 8a7d6719909..00000000000
--- a/src/test/regress/sql/appendonly_format.sql
+++ /dev/null
@@ -1,57 +0,0 @@
--- Test dealing with Append-only tables created with previous version
--- of GPDB.
---
--- You cannot have such old-format segments in a cluster that's been
--- initialized with the current version, but with pg_upgrade you can. To
--- keep things simple, we don't want to involve pg_upgrade in these tests,
--- however. Instead, hack the catalogs directly to get the same situation
--- that you would have after pg_upgrade.
-
-
--- GPDB 4.3 -> GPDB 5.0 numerics conversion.
---
--- The on-disk representation of numerics changed between PostgreSQL 8.2
--- and 8.3. The weight and scale fields in the NumericData struct were
--- swapped. Construct an old-format Datum, by creating a dummy cast from
--- bytea to numeric that allows us to treat any binary blob as numeric,
--- and insert that into a table. Finally, hack the pg_aoseg catalog to
--- change the version number.
---
--- Note: Numeric columns need stricter alignment than byteas. Having
--- an int4 before the numeric should do the trick.
-
--- also test with a domain over numeric
-CREATE DOMAIN numdomain AS numeric;
-
--- Construct the table, with the bytea
-CREATE TABLE numeric_ao_upgrade (astext text, numcol numeric, domcol numdomain) with (appendonly = true);
-
-
-begin;
-create cast (bytea as numeric) without function;
-
-INSERT INTO numeric_ao_upgrade VALUES
-  ('1234.5678 and 9.876543210', E'\\000\\000\\004\\000\\322\\004\\056\\026'::bytea::numeric, E'\\000\\000\\011\\000\\011\\000\\075\\042\\341\\020'::bytea::numeric);
-
--- Drop the dangerous cast.
-drop cast (bytea as numeric);
-commit;
-
-
-begin;
-set local allow_system_table_mods = true;
-
--- Hack the format version number in the pg_aoseg table. The name of the
--- table changes, so have to use dynamic SQL here.
-CREATE FUNCTION pg_temp.tmpfunc() returns void as $$
-begin
-  execute 'update pg_aoseg.pg_aoseg_' || 'numeric_ao_upgrade'::regclass::oid ||
-  	  ' set formatversion = 2';
-end;
-$$ language plpgsql;
-
-select pg_temp.tmpfunc();
-
-drop function pg_temp.tmpfunc();
-
-commit;
diff --git a/src/test/singlenode_isolation2/input/ao_upgrade.source b/src/test/singlenode_isolation2/input/ao_upgrade.source
deleted file mode 100644
index 7c4ab8535a8..00000000000
--- a/src/test/singlenode_isolation2/input/ao_upgrade.source
+++ /dev/null
@@ -1,86 +0,0 @@
--- set_ao_formatversion forces an AO[CO] format to a specific version (the last
--- argument is set to true for a column-oriented table, and false otherwise).
-CREATE OR REPLACE FUNCTION set_ao_formatversion(aosegrel oid, version smallint, isaocs bool)
-	RETURNS bool
-	AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'setAOFormatVersion'
-	LANGUAGE C
-	RETURNS NULL ON NULL INPUT;
-
-DROP TABLE IF EXISTS ao_upgrade_test;
-DROP TABLE IF EXISTS aocs_upgrade_test;
-DROP TABLE IF EXISTS aocs_rle_upgrade_test;
-
-CREATE TABLE ao_upgrade_test (rowid int, n numeric) USING ao_row;
-CREATE TABLE aocs_upgrade_test (rowid int, n numeric) USING ao_column;
-CREATE TABLE aocs_rle_upgrade_test (rowid int, n numeric) USING ao_column WITH (compresstype=RLE_TYPE);
-
--- We want to load GPDB4 numerics into the table; to do that, add a direct cast
--- from bytea to numeric so we can hardcode what the GPDB4 data looked like.
-CREATE CAST (bytea AS numeric) WITHOUT FUNCTION;
-
-INSERT INTO ao_upgrade_test VALUES
-	(1, '\x000003000c007a0d'::bytea::numeric),	-- 12.345
-	(2, '\x00000000'::bytea::numeric),			-- 0
-	(3, '\x000003400c007a0d'::bytea::numeric),	-- -12.345
-	(4, '\x010000000100'::bytea::numeric),		-- 10000
-	(5, '\xfeff0500e803'::bytea::numeric),		-- 0.00001
-	(6, '\xfeff0900e803'::bytea::numeric),		-- 0.000010000
-	(7, '\x190000000100'::bytea::numeric),		-- 1e100
-	(8, '\x010000002400400b'::bytea::numeric),	-- 9! (362880)
-	(9, '\x000000c0'::bytea::numeric);			-- NaN
-INSERT INTO aocs_upgrade_test VALUES
-	(1, '\x000003000c007a0d'::bytea::numeric),	-- 12.345
-	(2, '\x00000000'::bytea::numeric),			-- 0
-	(3, '\x000003400c007a0d'::bytea::numeric),	-- -12.345
-	(4, '\x010000000100'::bytea::numeric),		-- 10000
-	(5, '\xfeff0500e803'::bytea::numeric),		-- 0.00001
-	(6, '\xfeff0900e803'::bytea::numeric),		-- 0.000010000
-	(7, '\x190000000100'::bytea::numeric),		-- 1e100
-	(8, '\x010000002400400b'::bytea::numeric),	-- 9! (362880)
-	(9, '\x000000c0'::bytea::numeric);			-- NaN
-
--- For the RLE test case, insert a bunch of identical numerics so they will be
--- run-length compressed.
-INSERT INTO aocs_rle_upgrade_test (SELECT a, '\x010000002400400b'::bytea::numeric FROM generate_series(1, 10) a);
-
--- Downgrade to GPDB4 (AO version 2).
---start_ignore
-*U: SELECT set_ao_formatversion(
-		(SELECT segrelid FROM pg_appendonly WHERE relid = 'ao_upgrade_test'::regclass),
-		2::smallint, false);
-*U: SELECT set_ao_formatversion(
-		(SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_upgrade_test'::regclass),
-		2::smallint, true);
-*U: SELECT set_ao_formatversion(
-		(SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_rle_upgrade_test'::regclass),
-		2::smallint, true);
---end_ignore
-
--- Scan test. The numerics should be fixed again.
-SELECT * FROM ao_upgrade_test;
-SELECT * FROM aocs_upgrade_test;
-SELECT * FROM aocs_rle_upgrade_test;
-
--- Fetch test. To force fetches, we'll add bitmap indexes and disable sequential
--- scan.
-CREATE INDEX ao_bitmap_index ON ao_upgrade_test USING bitmap(n);
-CREATE INDEX aocs_bitmap_index ON aocs_upgrade_test USING bitmap(n);
-CREATE INDEX aocs_rle_bitmap_index ON aocs_rle_upgrade_test USING bitmap(n);
-
-SET enable_seqscan TO off;
-
--- Ensure we're using a bitmap scan for our tests. Upgrade note to developers:
--- the only thing that this test needs to verify is that a fetch-based scan is
--- in use. Other diffs are fine.
-EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = factorial(9);
-EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = factorial(9);
-EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9);
-
-SELECT n FROM ao_upgrade_test WHERE n = factorial(9);
-SELECT n FROM aocs_upgrade_test WHERE n = factorial(9);
-SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9);
-
-RESET enable_seqscan;
-
-DROP CAST (bytea AS numeric);
-DROP FUNCTION set_ao_formatversion(oid, smallint, bool);
diff --git a/src/test/singlenode_isolation2/isolation2_schedule b/src/test/singlenode_isolation2/isolation2_schedule
index f19caa8045c..7e5266d1c84 100644
--- a/src/test/singlenode_isolation2/isolation2_schedule
+++ b/src/test/singlenode_isolation2/isolation2_schedule
@@ -90,7 +90,6 @@ test: instr_in_shmem_terminate
 test: invalidated_toast_index
 # test: distributed_snapshot
 # test: gp_collation
-test: ao_upgrade
 # test: bitmap_index_concurrent
 # test: bitmap_index_crash
 # test: bitmap_update_words_backup_block
diff --git a/src/test/singlenode_isolation2/output/ao_upgrade.source b/src/test/singlenode_isolation2/output/ao_upgrade.source
deleted file mode 100644
index 5bfc56e95c3..00000000000
--- a/src/test/singlenode_isolation2/output/ao_upgrade.source
+++ /dev/null
@@ -1,205 +0,0 @@
--- set_ao_formatversion forces an AO[CO] format to a specific version (the last
--- argument is set to true for a column-oriented table, and false otherwise).
-CREATE OR REPLACE FUNCTION set_ao_formatversion(aosegrel oid, version smallint, isaocs bool) RETURNS bool AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'setAOFormatVersion' LANGUAGE C RETURNS NULL ON NULL INPUT;
-CREATE
-
-DROP TABLE IF EXISTS ao_upgrade_test;
-DROP
-DROP TABLE IF EXISTS aocs_upgrade_test;
-DROP
-DROP TABLE IF EXISTS aocs_rle_upgrade_test;
-DROP
-
-CREATE TABLE ao_upgrade_test (rowid int, n numeric) USING ao_row;
-CREATE
-CREATE TABLE aocs_upgrade_test (rowid int, n numeric) USING ao_column;
-CREATE
-CREATE TABLE aocs_rle_upgrade_test (rowid int, n numeric) USING ao_column WITH (compresstype=RLE_TYPE);
-CREATE
-
--- We want to load GPDB4 numerics into the table; to do that, add a direct cast
--- from bytea to numeric so we can hardcode what the GPDB4 data looked like.
-CREATE CAST (bytea AS numeric) WITHOUT FUNCTION;
-CREATE
-
-INSERT INTO ao_upgrade_test VALUES (1, '\x000003000c007a0d'::bytea::numeric),	-- 12.345 (2, '\x00000000'::bytea::numeric),			-- 0 (3, '\x000003400c007a0d'::bytea::numeric),	-- -12.345 (4, '\x010000000100'::bytea::numeric),		-- 10000 (5, '\xfeff0500e803'::bytea::numeric),		-- 0.00001 (6, '\xfeff0900e803'::bytea::numeric),		-- 0.000010000 (7, '\x190000000100'::bytea::numeric),		-- 1e100 (8, '\x010000002400400b'::bytea::numeric),	-- 9! (362880) (9, '\x000000c0'::bytea::numeric);			-- NaN INSERT INTO aocs_upgrade_test VALUES (1, '\x000003000c007a0d'::bytea::numeric),	-- 12.345 (2, '\x00000000'::bytea::numeric),			-- 0 (3, '\x000003400c007a0d'::bytea::numeric),	-- -12.345 (4, '\x010000000100'::bytea::numeric),		-- 10000 (5, '\xfeff0500e803'::bytea::numeric),		-- 0.00001 (6, '\xfeff0900e803'::bytea::numeric),		-- 0.000010000 (7, '\x190000000100'::bytea::numeric),		-- 1e100 (8, '\x010000002400400b'::bytea::numeric),	-- 9! (362880) (9, '\x000000c0'::bytea::numeric);			-- NaN 
--- For the RLE test case, insert a bunch of identical numerics so they will be
--- run-length compressed.
-INSERT INTO aocs_rle_upgrade_test (SELECT a, '\x010000002400400b'::bytea::numeric FROM generate_series(1, 10) a);
-INSERT 10
-
--- Downgrade to GPDB4 (AO version 2).
---start_ignore
-*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'ao_upgrade_test'::regclass), 2::smallint, false);
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_upgrade_test'::regclass), 2::smallint, true);
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_rle_upgrade_test'::regclass), 2::smallint, true);
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
-
- set_ao_formatversion 
-----------------------
- t                    
-(1 row)
---end_ignore
-
--- Scan test. The numerics should be fixed again.
-SELECT * FROM ao_upgrade_test;
- rowid | n                                                                                                     
--------+-------------------------------------------------------------------------------------------------------
- 2     | 0                                                                                                     
- 5     | 0.00001                                                                                               
- 6     | 0.000010000                                                                                           
- 9     | NaN                                                                                                   
- 1     | 12.345                                                                                                
- 3     | -12.345                                                                                               
- 4     | 10000                                                                                                 
- 7     | 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 
- 8     | 362880                                                                                                
-(9 rows)
-SELECT * FROM aocs_upgrade_test;
- rowid | n                                                                                                     
--------+-------------------------------------------------------------------------------------------------------
- 2     | 0                                                                                                     
- 5     | 0.00001                                                                                               
- 6     | 0.000010000                                                                                           
- 9     | NaN                                                                                                   
- 4     | 10000                                                                                                 
- 7     | 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 
- 8     | 362880                                                                                                
- 1     | 12.345                                                                                                
- 3     | -12.345                                                                                               
-(9 rows)
-SELECT * FROM aocs_rle_upgrade_test;
- rowid | n      
--------+--------
- 1     | 362880 
- 2     | 362880 
- 3     | 362880 
- 4     | 362880 
- 5     | 362880 
- 6     | 362880 
- 7     | 362880 
- 8     | 362880 
- 9     | 362880 
- 10    | 362880 
-(10 rows)
-
--- Fetch test. To force fetches, we'll add bitmap indexes and disable sequential
--- scan.
-CREATE INDEX ao_bitmap_index ON ao_upgrade_test USING bitmap(n);
-CREATE
-CREATE INDEX aocs_bitmap_index ON aocs_upgrade_test USING bitmap(n);
-CREATE
-CREATE INDEX aocs_rle_bitmap_index ON aocs_rle_upgrade_test USING bitmap(n);
-CREATE
-
-SET enable_seqscan TO off;
-SET
-
--- Ensure we're using a bitmap scan for our tests. Upgrade note to developers:
--- the only thing that this test needs to verify is that a fetch-based scan is
--- in use. Other diffs are fine.
-EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = factorial(9);
- QUERY PLAN                                                                             
-----------------------------------------------------------------------------------------
- Index Scan using ao_bitmap_index on ao_upgrade_test  (cost=0.00..8.02 rows=1 width=32) 
-   Index Cond: (n = '362880'::numeric)                                                  
- Optimizer: Postgres query optimizer                                                    
-(3 rows)
-EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = factorial(9);
- QUERY PLAN                                                                                 
---------------------------------------------------------------------------------------------
- Index Scan using aocs_bitmap_index on aocs_upgrade_test  (cost=0.00..8.02 rows=1 width=32) 
-   Index Cond: (n = '362880'::numeric)                                                      
- Optimizer: Postgres query optimizer                                                        
-(3 rows)
-EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9);
- QUERY PLAN                                                                                         
-----------------------------------------------------------------------------------------------------
- Index Scan using aocs_rle_bitmap_index on aocs_rle_upgrade_test  (cost=0.00..8.02 rows=1 width=32) 
-   Index Cond: (n = '362880'::numeric)                                                              
- Optimizer: Postgres query optimizer                                                                
-(3 rows)
-
-SELECT n FROM ao_upgrade_test WHERE n = factorial(9);
- n      
---------
- 362880 
-(1 row)
-SELECT n FROM aocs_upgrade_test WHERE n = factorial(9);
- n      
---------
- 362880 
-(1 row)
-SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9);
- n      
---------
- 362880 
- 362880 
- 362880 
- 362880 
- 362880 
- 362880 
- 362880 
- 362880 
- 362880 
- 362880 
-(10 rows)
-
-RESET enable_seqscan;
-RESET
-
-DROP CAST (bytea AS numeric);
-DROP
-DROP FUNCTION set_ao_formatversion(oid, smallint, bool);
-DROP


Reviews:
user: login: my-ship-it
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T11:54:15Z
user: login: reshke
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T12:21:18Z
user: login: yjhjstz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T14:17:52Z

Review comments:
diff_hunk: @@ -675,82 +675,140 @@ aocs_endscan(AOCSScanDesc scan)
 	pfree(scan);
 }
 
-/*
- * Upgrades a Datum value from a previous version of the AOCS page format. The
- * DatumStreamRead that is passed must correspond to the column being upgraded.
- */
-static void upgrade_datum_impl(DatumStreamRead *ds, int attno, Datum values[],
-							   bool isnull[], int formatversion)
+
+static pg_attribute_hot_inline bool
+aocs_getnext_noqual(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
-	bool 	convert_numeric = false;
+	Datum	   *d = slot->tts_values;
+	bool	   *null = slot->tts_isnull;
+	AOTupleId	aoTupleId;
+	int64		rowNum = INT64CONST(-1);
+	int			err = 0;
+	bool		isSnapshotAny = (scan->rs_base.rs_snapshot == SnapshotAny);
+	AttrNumber	natts;
 
-	if (PG82NumericConversionNeeded(formatversion))
+	Assert(ScanDirectionIsForward(direction));
+
+	if (scan->columnScanInfo.relationTupleDesc == NULL)
 	{
-		/*
-		 * On the first call for this DatumStream, figure out if this column is
-		 * a numeric, or a domain over numerics.
-		 *
-		 * TODO: consolidate this code with upgrade_tuple() in appendonlyam.c.
-		 */
-		if (!OidIsValid(ds->baseTypeOid))
+		scan->columnScanInfo.relationTupleDesc = slot->tts_tupleDescriptor;
+		/* Pin it! ... and of course release it upon destruction / rescan */
+		PinTupleDesc(scan->columnScanInfo.relationTupleDesc);
+		initscan_with_colinfo(scan);
+	}
+
+	natts = slot->tts_tupleDescriptor->natts;
+	Assert(natts <= scan->columnScanInfo.relationTupleDesc->natts);
+
+	while (1)
+	{
+		AOCSFileSegInfo *curseginfo;
+		bool visible_pass;
+
+ReadNext:
+		/* If necessary, open next seg */
+		if (scan->cur_seg < 0 || err < 0)
 		{
-			ds->baseTypeOid = getBaseType(ds->typeInfo.typid);
+			err = open_next_scan_seg(scan);
+			if (err < 0)
+			{
+				/* No more seg, we are at the end */
+				ExecClearTuple(slot);
+				scan->cur_seg = -1;
+				return false;
+			}
+			scan->segrowsprocessed = 0;
 		}
 
-		/* If this Datum is a numeric, we need to convert it. */
-		convert_numeric = (ds->baseTypeOid == NUMERICOID) && !isnull[attno];
-	}
+		Assert(scan->cur_seg >= 0);
+		curseginfo = scan->seginfo[scan->cur_seg];
 
-	if (convert_numeric)
-	{
-		/*
-		 * Before PostgreSQL 8.3, the n_weight and n_sign_dscale fields were the
-		 * other way 'round. Swap them.
-		 */
-		Datum 		datum;
-		char	   *numericdata;
-		char	   *upgradedata;
-		size_t		datalen;
-		uint16		tmp;
+		/* Read from cur_seg */
+		visible_pass = true;
+		for (AttrNumber i = 0; i < scan->columnScanInfo.num_proj_atts; i++)
+		{
+			AttrNumber	attno = scan->columnScanInfo.proj_atts[i];
 
-		/*
-		 * We need to make a copy of this data so that any other tuples pointing
-		 * to it won't be affected. Store it in the upgrade space for this
-		 * DatumStream.
-		 */
-		datum = values[attno];
-		datalen = VARSIZE_ANY(DatumGetPointer(datum));
+			err = datumstreamread_advance(scan->columnScanInfo.ds[attno]);
+			Assert(err >= 0);
+			if (err == 0)
+			{
+				err = datumstreamread_block(scan->columnScanInfo.ds[attno], scan->blockDirectory, attno);
+				if (err < 0)
+				{
+					/*
+					 * Ha, cannot read next block, we need to go to next seg
+					 */
+					close_cur_scan_seg(scan);
+					goto ReadNext;
+				}
 
-		upgradedata = datumstreamread_get_upgrade_space(ds, datalen);
-		memcpy(upgradedata, DatumGetPointer(datum), datalen);
+				AOCSScanDesc_UpdateTotalBytesRead(scan, attno);
 
-		/* Swap the fields. */
-		numericdata = VARDATA_ANY(upgradedata);
+				err = datumstreamread_advance(scan->columnScanInfo.ds[attno]);
+				Assert(err > 0);
+			}
+			if (!visible_pass)
+				continue; /* not break, need advance for other cols */
 
-		memcpy(&tmp, &numericdata[0], 2);
-		memcpy(&numericdata[0], &numericdata[2], 2);
-		memcpy(&numericdata[2], &tmp, 2);
+			/*
+			 * Get the column's datum right here since the data structures
+			 * should still be hot in CPU data cache memory.
+			 */
+			datumstreamread_get(scan->columnScanInfo.ds[attno], &d[attno], &null[attno]);
 
-		/* Re-point the Datum to the upgraded numeric. */
-		values[attno] = PointerGetDatum(upgradedata);
-	}
-}
+			if (i == 0)
+			{
+				if (rowNum == INT64CONST(-1) &&
+					scan->columnScanInfo.ds[attno]->blockFirstRowNum != INT64CONST(-1))
+				{
+					Assert(scan->columnScanInfo.ds[attno]->blockFirstRowNum > 0);
+					rowNum = scan->columnScanInfo.ds[attno]->blockFirstRowNum +
+						datumstreamread_nth(scan->columnScanInfo.ds[attno]);
+				}
+				scan->segrowsprocessed++;
+				if (rowNum == INT64CONST(-1))
+				{
+					AOTupleIdInit(&aoTupleId, curseginfo->segno, scan->segrowsprocessed);
+				}
+				else
+				{
+					AOTupleIdInit(&aoTupleId, curseginfo->segno, rowNum);
+				}
 
-static void upgrade_datum_scan(AOCSScanDesc scan, int attno, Datum values[],
-							   bool isnull[], int formatversion)
-{
-	upgrade_datum_impl(scan->columnScanInfo.ds[attno], attno, values, isnull, formatversion);
-}
+				if (!isSnapshotAny && !AppendOnlyVisimap_IsVisible(&scan->visibilityMap, &aoTupleId))
+				{
+					/*
+					 * The tuple is invisible.
+					 * In `analyze`, we can simply return false
+					 */
+					if ((scan->rs_base.rs_flags & SO_TYPE_ANALYZE) != 0)
+						return false;
 
-static void upgrade_datum_fetch(AOCSFetchDesc fetch, int attno, Datum values[],
-								bool isnull[], int formatversion)
-{
-	upgrade_datum_impl(fetch->datumStreamFetchDesc[attno]->datumStream, attno,
-					   values, isnull, formatversion);
+					rowNum = INT64CONST(-1);
+					visible_pass = false;
+					continue; /* not break, need advance for other cols */
+				}
+			}
+		}
+		if (!visible_pass)
+		{
+			rowNum = INT64CONST(-1);
+			goto ReadNext;
+		}
+		scan->cdb_fake_ctid = *((ItemPointer) &aoTupleId);
+
+		slot->tts_nvalid = natts;
+		slot->tts_tid = scan->cdb_fake_ctid;
+		return true;
+	}
+
+	Assert(!"Never here");
+	return false;
 }
 
-bool
-aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
+static bool
+aocs_getnext_withqual(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
 	if (scan->aos_pushdown_qual && scan->aos_scaned_rows < scan->aos_sample_rows)
 	{
path: src/backend/access/aocs/aocsam.c
user: login: my-ship-it
body: should we call aocs_getnext_noqual later?
created_at: 2025-03-27T11:51:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -675,82 +675,140 @@ aocs_endscan(AOCSScanDesc scan)
 	pfree(scan);
 }
 
-/*
- * Upgrades a Datum value from a previous version of the AOCS page format. The
- * DatumStreamRead that is passed must correspond to the column being upgraded.
- */
-static void upgrade_datum_impl(DatumStreamRead *ds, int attno, Datum values[],
-							   bool isnull[], int formatversion)
+
+static pg_attribute_hot_inline bool
+aocs_getnext_noqual(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
-	bool 	convert_numeric = false;
+	Datum	   *d = slot->tts_values;
+	bool	   *null = slot->tts_isnull;
+	AOTupleId	aoTupleId;
+	int64		rowNum = INT64CONST(-1);
+	int			err = 0;
+	bool		isSnapshotAny = (scan->rs_base.rs_snapshot == SnapshotAny);
+	AttrNumber	natts;
 
-	if (PG82NumericConversionNeeded(formatversion))
+	Assert(ScanDirectionIsForward(direction));
+
+	if (scan->columnScanInfo.relationTupleDesc == NULL)
 	{
-		/*
-		 * On the first call for this DatumStream, figure out if this column is
-		 * a numeric, or a domain over numerics.
-		 *
-		 * TODO: consolidate this code with upgrade_tuple() in appendonlyam.c.
-		 */
-		if (!OidIsValid(ds->baseTypeOid))
+		scan->columnScanInfo.relationTupleDesc = slot->tts_tupleDescriptor;
+		/* Pin it! ... and of course release it upon destruction / rescan */
+		PinTupleDesc(scan->columnScanInfo.relationTupleDesc);
+		initscan_with_colinfo(scan);
+	}
+
+	natts = slot->tts_tupleDescriptor->natts;
+	Assert(natts <= scan->columnScanInfo.relationTupleDesc->natts);
+
+	while (1)
+	{
+		AOCSFileSegInfo *curseginfo;
+		bool visible_pass;
+
+ReadNext:
+		/* If necessary, open next seg */
+		if (scan->cur_seg < 0 || err < 0)
 		{
-			ds->baseTypeOid = getBaseType(ds->typeInfo.typid);
+			err = open_next_scan_seg(scan);
+			if (err < 0)
+			{
+				/* No more seg, we are at the end */
+				ExecClearTuple(slot);
+				scan->cur_seg = -1;
+				return false;
+			}
+			scan->segrowsprocessed = 0;
 		}
 
-		/* If this Datum is a numeric, we need to convert it. */
-		convert_numeric = (ds->baseTypeOid == NUMERICOID) && !isnull[attno];
-	}
+		Assert(scan->cur_seg >= 0);
+		curseginfo = scan->seginfo[scan->cur_seg];
 
-	if (convert_numeric)
-	{
-		/*
-		 * Before PostgreSQL 8.3, the n_weight and n_sign_dscale fields were the
-		 * other way 'round. Swap them.
-		 */
-		Datum 		datum;
-		char	   *numericdata;
-		char	   *upgradedata;
-		size_t		datalen;
-		uint16		tmp;
+		/* Read from cur_seg */
+		visible_pass = true;
+		for (AttrNumber i = 0; i < scan->columnScanInfo.num_proj_atts; i++)
+		{
+			AttrNumber	attno = scan->columnScanInfo.proj_atts[i];
 
-		/*
-		 * We need to make a copy of this data so that any other tuples pointing
-		 * to it won't be affected. Store it in the upgrade space for this
-		 * DatumStream.
-		 */
-		datum = values[attno];
-		datalen = VARSIZE_ANY(DatumGetPointer(datum));
+			err = datumstreamread_advance(scan->columnScanInfo.ds[attno]);
+			Assert(err >= 0);
+			if (err == 0)
+			{
+				err = datumstreamread_block(scan->columnScanInfo.ds[attno], scan->blockDirectory, attno);
+				if (err < 0)
+				{
+					/*
+					 * Ha, cannot read next block, we need to go to next seg
+					 */
+					close_cur_scan_seg(scan);
+					goto ReadNext;
+				}
 
-		upgradedata = datumstreamread_get_upgrade_space(ds, datalen);
-		memcpy(upgradedata, DatumGetPointer(datum), datalen);
+				AOCSScanDesc_UpdateTotalBytesRead(scan, attno);
 
-		/* Swap the fields. */
-		numericdata = VARDATA_ANY(upgradedata);
+				err = datumstreamread_advance(scan->columnScanInfo.ds[attno]);
+				Assert(err > 0);
+			}
+			if (!visible_pass)
+				continue; /* not break, need advance for other cols */
 
-		memcpy(&tmp, &numericdata[0], 2);
-		memcpy(&numericdata[0], &numericdata[2], 2);
-		memcpy(&numericdata[2], &tmp, 2);
+			/*
+			 * Get the column's datum right here since the data structures
+			 * should still be hot in CPU data cache memory.
+			 */
+			datumstreamread_get(scan->columnScanInfo.ds[attno], &d[attno], &null[attno]);
 
-		/* Re-point the Datum to the upgraded numeric. */
-		values[attno] = PointerGetDatum(upgradedata);
-	}
-}
+			if (i == 0)
+			{
+				if (rowNum == INT64CONST(-1) &&
+					scan->columnScanInfo.ds[attno]->blockFirstRowNum != INT64CONST(-1))
+				{
+					Assert(scan->columnScanInfo.ds[attno]->blockFirstRowNum > 0);
+					rowNum = scan->columnScanInfo.ds[attno]->blockFirstRowNum +
+						datumstreamread_nth(scan->columnScanInfo.ds[attno]);
+				}
+				scan->segrowsprocessed++;
+				if (rowNum == INT64CONST(-1))
+				{
+					AOTupleIdInit(&aoTupleId, curseginfo->segno, scan->segrowsprocessed);
+				}
+				else
+				{
+					AOTupleIdInit(&aoTupleId, curseginfo->segno, rowNum);
+				}
 
-static void upgrade_datum_scan(AOCSScanDesc scan, int attno, Datum values[],
-							   bool isnull[], int formatversion)
-{
-	upgrade_datum_impl(scan->columnScanInfo.ds[attno], attno, values, isnull, formatversion);
-}
+				if (!isSnapshotAny && !AppendOnlyVisimap_IsVisible(&scan->visibilityMap, &aoTupleId))
+				{
+					/*
+					 * The tuple is invisible.
+					 * In `analyze`, we can simply return false
+					 */
+					if ((scan->rs_base.rs_flags & SO_TYPE_ANALYZE) != 0)
+						return false;
 
-static void upgrade_datum_fetch(AOCSFetchDesc fetch, int attno, Datum values[],
-								bool isnull[], int formatversion)
-{
-	upgrade_datum_impl(fetch->datumStreamFetchDesc[attno]->datumStream, attno,
-					   values, isnull, formatversion);
+					rowNum = INT64CONST(-1);
+					visible_pass = false;
+					continue; /* not break, need advance for other cols */
+				}
+			}
+		}
+		if (!visible_pass)
+		{
+			rowNum = INT64CONST(-1);
+			goto ReadNext;
+		}
+		scan->cdb_fake_ctid = *((ItemPointer) &aoTupleId);
+
+		slot->tts_nvalid = natts;
+		slot->tts_tid = scan->cdb_fake_ctid;
+		return true;
+	}
+
+	Assert(!"Never here");
+	return false;
 }
 
-bool
-aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
+static bool
+aocs_getnext_withqual(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
 	if (scan->aos_pushdown_qual && scan->aos_scaned_rows < scan->aos_sample_rows)
 	{
path: src/backend/access/aocs/aocsam.c
user: login: yjhjstz
body: still aocs_getnext()
created_at: 2025-03-27T14:17:52Z
author_association: MEMBER

PR comments:
user: login: my-ship-it
created_at: 2025-03-27T11:54:35Z
body: Nice catch!
author_association: CONTRIBUTOR
user: login: reshke
created_at: 2025-03-27T12:21:31Z
body: Do we have bench here?
author_association: CONTRIBUTOR
user: login: yjhjstz
created_at: 2025-03-27T14:18:29Z
body: > Do we have bench here?

Bench by tpcds, shows much improvement.
author_association: MEMBER