Repository: timescale/timescaledb
Pull request #8089: Split continuous aggregate invalidation state
Author: login: mkindahl
Labels: name: Continuous Aggregate
name: refactor
name: released-2.21.0
description: Released in 2.21.0
Created: 2025-05-09T09:14:57Z | Merged: 2025-05-13T08:49:33Z

We have two separate invalidation states for the two different phases of the continuous aggregate invalidation, so this commit split the invalidation state into two different states to make it clear what belongs to which phase. This is a precursor to separating the two phases completely and allowing them to run independently.

Disable-check: force-changelog-file


Diff:
diff --git a/tsl/src/continuous_aggs/invalidation.c b/tsl/src/continuous_aggs/invalidation.c
index 2139134376d..f637a63d14e 100644
--- a/tsl/src/continuous_aggs/invalidation.c
+++ b/tsl/src/continuous_aggs/invalidation.c
@@ -28,7 +28,6 @@
 #include <time_utils.h>
 #include <utils.h>
 
-#include "compat/compat.h"
 #include "continuous_aggs/materialize.h"
 #include "invalidation.h"
 #include "refresh.h"
@@ -86,19 +85,35 @@
  * defined by the refresh window.
  */
 
-typedef struct CaggInvalidationState
+/*
+ * Processing state used while processing the materialization invalidation log
+ * and refreshing the continuous aggregate.
+ */
+typedef struct CAggInvalidationState
 {
 	int32 mat_hypertable_id;
-	int32 raw_hypertable_id;
 	Oid dimtype; /* The type of the underlying hypertable's time dimension
 				  * that is bucketed */
 	MemoryContext per_tuple_mctx;
 	Relation cagg_log_rel;
 	Snapshot snapshot;
 	Tuplestorestate *invalidations;
-	const CaggsInfo *all_caggs;
 	const ContinuousAggsBucketFunction *bucket_function;
-} CaggInvalidationState;
+} CAggInvalidationState;
+
+/*
+ * Processing state used while moving invalidations from hypertable
+ * invalidation log to materialization invalidation log.
+ */
+typedef struct HypertableInvalidationState
+{
+	int32 hypertable_id;
+	Oid dimtype; /* Type of the underlying hypertable's bucketed attribute */
+	const CaggsInfo *all_caggs;
+	MemoryContext per_tuple_mctx;
+	Relation cagg_log_rel;
+	Snapshot snapshot;
+} HypertableInvalidationState;
 
 typedef enum LogType
 {
@@ -111,7 +126,7 @@ static void hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hype
 											  LOCKMODE lockmode);
 static HeapTuple create_invalidation_tup(const TupleDesc tupdesc, int32 cagg_hyper_id, int64 start,
 										 int64 end);
-static bool save_invalidation_for_refresh(const CaggInvalidationState *state,
+static bool save_invalidation_for_refresh(const CAggInvalidationState *state,
 										  const Invalidation *invalidation);
 static void set_remainder_after_cut(Invalidation *remainder, int32 hyper_id,
 									int64 lowest_modified_value, int64 greatest_modified_value);
@@ -128,23 +143,23 @@ invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleIn
 											  const ContinuousAggsBucketFunction *bucket_function);
 static bool invalidations_can_be_merged(const Invalidation *a, const Invalidation *b);
 static bool invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry);
-static void cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state,
+static void cut_and_insert_new_cagg_invalidation(const HypertableInvalidationState *state,
 												 const Invalidation *entry, int32 cagg_hyper_id);
-static void move_invalidations_from_hyper_to_cagg_log(const CaggInvalidationState *state);
+static void move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state);
 static void cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id,
 													   LOCKMODE lockmode);
-static Invalidation cut_cagg_invalidation(const CaggInvalidationState *state,
+static Invalidation cut_cagg_invalidation(const CAggInvalidationState *state,
 										  const InternalTimeRange *refresh_window,
 										  const Invalidation *entry);
 static Invalidation cut_cagg_invalidation_and_compute_remainder(
-	const CaggInvalidationState *state, const InternalTimeRange *refresh_window,
+	const CAggInvalidationState *state, const InternalTimeRange *refresh_window,
 	const Invalidation *mergedentry, const Invalidation *current_remainder);
-static void clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state,
+static void clear_cagg_invalidations_for_refresh(const CAggInvalidationState *state,
 												 const InternalTimeRange *refresh_window,
 												 bool force);
-static void invalidation_state_init(CaggInvalidationState *state, const ContinuousAgg *cagg,
-									Oid dimtype, const CaggsInfo *all_caggs);
-static void invalidation_state_cleanup(const CaggInvalidationState *state);
+static void cagg_invalidation_state_init(CAggInvalidationState *state, const ContinuousAgg *cagg,
+										 Oid dimtype);
+static void cagg_invalidation_state_cleanup(const CAggInvalidationState *state);
 
 static Relation
 open_invalidation_log(LogType type, LOCKMODE lockmode)
@@ -283,7 +298,7 @@ typedef enum InvalidationResult
 #define IS_VALID_INVALIDATION(entry) ((entry)->hyper_id > 0)
 
 static bool
-save_invalidation_for_refresh(const CaggInvalidationState *state, const Invalidation *invalidation)
+save_invalidation_for_refresh(const CAggInvalidationState *state, const Invalidation *invalidation)
 {
 	int32 cagg_hyper_id = state->mat_hypertable_id;
 	TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel);
@@ -326,7 +341,7 @@ set_remainder_after_cut(Invalidation *remainder, int32 hyper_id, int64 lowest_mo
  * are inclusive.
  */
 static InvalidationResult
-cut_invalidation_along_refresh_window(const CaggInvalidationState *state,
+cut_invalidation_along_refresh_window(const CAggInvalidationState *state,
 									  const Invalidation *invalidation,
 									  const InternalTimeRange *refresh_window,
 									  Invalidation *remainder)
@@ -659,8 +674,8 @@ invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry)
 }
 
 static void
-cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state, const Invalidation *entry,
-									 int32 cagg_hyper_id)
+cut_and_insert_new_cagg_invalidation(const HypertableInvalidationState *state,
+									 const Invalidation *entry, int32 cagg_hyper_id)
 {
 	CatalogSecurityContext sec_ctx;
 	TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel);
@@ -690,10 +705,10 @@ cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state, const I
  * invalidation log.
  */
 static void
-move_invalidations_from_hyper_to_cagg_log(const CaggInvalidationState *state)
+move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state)
 {
 	const CaggsInfo *all_caggs = state->all_caggs;
-	int32 hyper_id = state->raw_hypertable_id;
+	int32 hyper_id = state->hypertable_id;
 	int32 last_cagg_hyper_id;
 	ListCell *lc1, *lc2;
 
@@ -796,7 +811,7 @@ cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hy
  * refresh window.
  */
 static Invalidation
-cut_cagg_invalidation(const CaggInvalidationState *state, const InternalTimeRange *refresh_window,
+cut_cagg_invalidation(const CAggInvalidationState *state, const InternalTimeRange *refresh_window,
 					  const Invalidation *entry)
 {
 	InvalidationResult result;
@@ -836,7 +851,7 @@ cut_cagg_invalidation(const CaggInvalidationState *state, const InternalTimeRang
 }
 
 static Invalidation
-cut_cagg_invalidation_and_compute_remainder(const CaggInvalidationState *state,
+cut_cagg_invalidation_and_compute_remainder(const CAggInvalidationState *state,
 											const InternalTimeRange *refresh_window,
 											const Invalidation *mergedentry,
 											const Invalidation *current_remainder)
@@ -878,7 +893,7 @@ cut_cagg_invalidation_and_compute_remainder(const CaggInvalidationState *state,
  * lowest_modified_value.
  */
 static void
-clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state,
+clear_cagg_invalidations_for_refresh(const CAggInvalidationState *state,
 									 const InternalTimeRange *refresh_window, bool force)
 {
 	ScanIterator iterator;
@@ -958,53 +973,77 @@ clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state,
 }
 
 static void
-invalidation_state_init(CaggInvalidationState *state, const ContinuousAgg *cagg, Oid dimtype,
-						const CaggsInfo *all_caggs)
+cagg_invalidation_state_init(CAggInvalidationState *state, const ContinuousAgg *cagg, Oid dimtype)
 {
-	state->bucket_function = cagg->bucket_function;
 	state->mat_hypertable_id = cagg->data.mat_hypertable_id;
-	state->raw_hypertable_id = cagg->data.raw_hypertable_id;
+	state->bucket_function = cagg->bucket_function;
+	state->dimtype = dimtype;
+	state->cagg_log_rel = open_invalidation_log(LOG_CAGG, RowExclusiveLock);
+	state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext,
+												  "Materialization invalidations",
+												  ALLOCSET_DEFAULT_SIZES);
+	state->snapshot = RegisterSnapshot(GetTransactionSnapshot());
+}
+
+static void
+cagg_invalidation_state_cleanup(const CAggInvalidationState *state)
+{
+	table_close(state->cagg_log_rel, NoLock);
+	UnregisterSnapshot(state->snapshot);
+	MemoryContextDelete(state->per_tuple_mctx);
+}
+
+static void
+hypertable_invalidation_state_init(HypertableInvalidationState *state, int32 hypertable_id,
+								   Oid dimtype, const CaggsInfo *all_caggs)
+{
+	state->hypertable_id = hypertable_id;
 	state->dimtype = dimtype;
 	state->all_caggs = all_caggs;
 	state->cagg_log_rel = open_invalidation_log(LOG_CAGG, RowExclusiveLock);
 	state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext,
-												  "Continuous aggregate invalidations",
+												  "Hypertable invalidations",
 												  ALLOCSET_DEFAULT_SIZES);
 	state->snapshot = RegisterSnapshot(GetTransactionSnapshot());
 }
 
 static void
-invalidation_state_cleanup(const CaggInvalidationState *state)
+hypertable_invalidation_state_cleanup(const HypertableInvalidationState *state)
 {
 	table_close(state->cagg_log_rel, NoLock);
 	UnregisterSnapshot(state->snapshot);
 	MemoryContextDelete(state->per_tuple_mctx);
 }
 
+/*
+ * Move invalidations from hypertable invalidation log to materialization
+ * invalidation log. This will move *all* hypertable invalidations to the
+ * connected continuous aggregates.
+ */
 void
-invalidation_process_hypertable_log(const ContinuousAgg *cagg, Oid dimtype,
-									const CaggsInfo *all_caggs)
+invalidation_process_hypertable_log(int32 hypertable_id, Oid dimtype)
 {
-	CaggInvalidationState state;
+	HypertableInvalidationState state;
+	const CaggsInfo all_caggs = ts_continuous_agg_get_all_caggs_info(hypertable_id);
 
-	invalidation_state_init(&state, cagg, dimtype, all_caggs);
+	hypertable_invalidation_state_init(&state, hypertable_id, dimtype, &all_caggs);
 	move_invalidations_from_hyper_to_cagg_log(&state);
-	invalidation_state_cleanup(&state);
+	hypertable_invalidation_state_cleanup(&state);
 }
 
 InvalidationStore *
 invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window,
-							  const CaggsInfo *all_caggs_info, const long max_materializations,
-							  bool *do_merged_refresh, InternalTimeRange *ret_merged_refresh_window,
-							  const CaggRefreshContext context, bool force)
+							  long max_materializations, bool *do_merged_refresh,
+							  InternalTimeRange *ret_merged_refresh_window,
+							  CaggRefreshContext context, bool force)
 {
-	CaggInvalidationState state;
+	CAggInvalidationState state;
 	InvalidationStore *store = NULL;
 	long count;
 
 	*do_merged_refresh = false;
 
-	invalidation_state_init(&state, cagg, refresh_window->type, all_caggs_info);
+	cagg_invalidation_state_init(&state, cagg, refresh_window->type);
 	state.invalidations = tuplestore_begin_heap(false, false, work_mem);
 	clear_cagg_invalidations_for_refresh(&state, refresh_window, force);
 	count = tuplestore_tuple_count(state.invalidations);
@@ -1020,7 +1059,7 @@ invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange
 		store->tupdesc = CreateTupleDescCopy(RelationGetDescr(state.cagg_log_rel));
 	}
 
-	invalidation_state_cleanup(&state);
+	cagg_invalidation_state_cleanup(&state);
 
 	/*
 	 * If there are many individual invalidation ranges to refresh, then
diff --git a/tsl/src/continuous_aggs/invalidation.h b/tsl/src/continuous_aggs/invalidation.h
index 7e19ab87b48..9810e47d703 100644
--- a/tsl/src/continuous_aggs/invalidation.h
+++ b/tsl/src/continuous_aggs/invalidation.h
@@ -42,13 +42,12 @@ extern void continuous_agg_invalidate_raw_ht(const Hypertable *raw_ht, int64 sta
 extern void continuous_agg_invalidate_mat_ht(const Hypertable *raw_ht, const Hypertable *mat_ht,
 											 int64 start, int64 end);
 
-extern void invalidation_process_hypertable_log(const ContinuousAgg *cagg, Oid dimtype,
-												const CaggsInfo *all_caggs_info);
+extern void invalidation_process_hypertable_log(int32 hypertable_id, Oid dimtype);
 
 extern InvalidationStore *
 invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window,
-							  const CaggsInfo *all_caggs_info, const long max_materializations,
-							  bool *do_merged_refresh, InternalTimeRange *ret_merged_refresh_window,
-							  const CaggRefreshContext context, bool force);
+							  long max_materializations, bool *do_merged_refresh,
+							  InternalTimeRange *ret_merged_refresh_window,
+							  CaggRefreshContext context, bool force);
 
 extern void invalidation_store_free(InvalidationStore *store);
diff --git a/tsl/src/continuous_aggs/refresh.c b/tsl/src/continuous_aggs/refresh.c
index 444cf040ba0..bc40fbadf3d 100644
--- a/tsl/src/continuous_aggs/refresh.c
+++ b/tsl/src/continuous_aggs/refresh.c
@@ -22,7 +22,6 @@
 #include "dimension_slice.h"
 #include "guc.h"
 #include "hypertable.h"
-#include "hypertable_cache.h"
 #include "invalidation.h"
 #include "invalidation_threshold.h"
 #include "materialize.h"
@@ -32,7 +31,6 @@
 #include "time_utils.h"
 #include "ts_catalog/catalog.h"
 #include "ts_catalog/continuous_agg.h"
-#include "utils.h"
 
 #define CAGG_REFRESH_LOG_LEVEL                                                                     \
 	(context.callctx == CAGG_REFRESH_POLICY || context.callctx == CAGG_REFRESH_POLICY_BATCHED ?    \
@@ -740,11 +738,8 @@ process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg,
 	 * windows.
 	 */
 	LockRelationOid(hyper_relid, ExclusiveLock);
-	const CaggsInfo all_caggs_info =
-		ts_continuous_agg_get_all_caggs_info(cagg->data.raw_hypertable_id);
 	invalidations = invalidation_process_cagg_log(cagg,
 												  refresh_window,
-												  &all_caggs_info,
 												  ts_guc_cagg_max_individual_materializations,
 												  &do_merged_refresh,
 												  &merged_refresh_window,
@@ -900,10 +895,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg,
 		return;
 	}
 
-	/* Process invalidations in the hypertable invalidation log */
-	const CaggsInfo all_caggs_info =
-		ts_continuous_agg_get_all_caggs_info(cagg->data.raw_hypertable_id);
-	invalidation_process_hypertable_log(cagg, refresh_window.type, &all_caggs_info);
+	invalidation_process_hypertable_log(cagg->data.raw_hypertable_id, refresh_window.type);
 
 	/* Commit and Start a new transaction */
 	SPI_commit_and_chain();


Reviews:
user: login: fabriziomello
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T13:46:49Z
user: login: fabriziomello
body: LGTM. The only thing is make clear in the commit/pr messages that this "invalidation state" is about Continuous Aggregate. For us is obvious but for new engineers and/or other user maybe it's not.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T13:48:39Z
user: login: melihmutlu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T14:55:09Z
user: login: fabriziomello
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T20:00:00Z
user: login: mkindahl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T07:06:21Z
user: login: melihmutlu
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-13T07:42:12Z

Review comments:
diff_hunk: @@ -86,20 +85,36 @@
  * defined by the refresh window.
  */
 
+/*
+ * Processing state used while processing the materialization invalidation log
+ * and refreshing the continuous aggregate.
+ */
 typedef struct CaggInvalidationState
path: tsl/src/continuous_aggs/invalidation.c
user: login: fabriziomello
body: ```suggestion
typedef struct CAggInvalidationState
```
Nit.
created_at: 2025-05-09T13:46:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -659,8 +674,8 @@ invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry)
 }
 
 static void
-cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state, const Invalidation *entry,
-									 int32 cagg_hyper_id)
+cut_and_insert_new_cagg_invalidation(const HypertableInvalidationState *state,
path: tsl/src/continuous_aggs/invalidation.c
user: login: melihmutlu
body: I'm a bit confused by the name of this function and wonder if it still implies what this function is supposed to do. First, would it make more sense to have `CAggInvalidationState` as first parameter instead of `HypertableInvalidationState`, similar to `cut_cagg_invalidation`? I see that only `state->cagg_log_rel` is needed and that can work with `CAggInvalidationState` too. But I don't really know much about the places where this function is called, so `HypertableInvalidationState` may make more sense depending on the context.

Also, does it really cut an invalidation? 
created_at: 2025-05-12T14:55:08Z
author_association: MEMBER
diff_hunk: @@ -659,8 +674,8 @@ invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry)
 }
 
 static void
-cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state, const Invalidation *entry,
-									 int32 cagg_hyper_id)
+cut_and_insert_new_cagg_invalidation(const HypertableInvalidationState *state,
path: tsl/src/continuous_aggs/invalidation.c
user: login: fabriziomello
body: The state here is the `HypertableIvalidationState` because we're moving the hypertable invalidation logs to the materialization invalidation logs. See the caller.
created_at: 2025-05-12T19:59:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -659,8 +674,8 @@ invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry)
 }
 
 static void
-cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state, const Invalidation *entry,
-									 int32 cagg_hyper_id)
+cut_and_insert_new_cagg_invalidation(const HypertableInvalidationState *state,
path: tsl/src/continuous_aggs/invalidation.c
user: login: mkindahl
body: The name is a bit strange since it does not really cut anything, as you say; it just inserts the entry into the materialization log. But as @fabriziomello says, this belongs to the first phase of the refresh (moving hypertable invalidations to materialization invalidations), so it should use `HypertableInvalidationState`.
created_at: 2025-05-13T07:06:20Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-05-09T09:30:32Z
body: ## [Codecov](https://app.codecov.io/gh/timescale/timescaledb/pull/8089?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=timescale) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 82.37%. Comparing base [(`534732c`)](https://app.codecov.io/gh/timescale/timescaledb/commit/534732c8053aa7be7c654f63751042e239f28ce5?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=timescale) to head [(`0889a9b`)](https://app.codecov.io/gh/timescale/timescaledb/commit/0889a9bfa6e75aba898287c7a92a6aeb5305168d?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=timescale).
> Report is 146 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #8089      +/-   ##
==========================================
+ Coverage   82.28%   82.37%   +0.08%     
==========================================
  Files         255      255              
  Lines       47656    47638      -18     
  Branches    12011    12007       -4     
==========================================
+ Hits        39214    39242      +28     
- Misses       3633     3650      +17     
+ Partials     4809     4746      -63     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/timescale/timescaledb/pull/8089?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=timescale).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=timescale).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: mkindahl
created_at: 2025-05-12T13:51:16Z
body: > LGTM. The only thing is make clear in the commit/pr messages that this "invalidation state" is about Continuous Aggregate. For us is obvious but for new engineers and/or other user maybe it's not.

Hmmmm... "Split continuous aggregate invalidation state"?
author_association: CONTRIBUTOR