Repository: PostHog/posthog
Pull request #30425: feat(propdefs): Isolate and vectorize batch writes in property-defs-rs
Author: login: eli-r-ph
Created: 2025-03-25T22:47:34Z | Merged: 2025-03-28T17:57:06Z

## Problem
Batch writes mix heterogenous data types and spread writes across multiple tables. 

## Changes
* Vectorize batch writes - [the internet](https://www.timescale.com/blog/boosting-postgres-insert-performance) thinks this will be a 2x query latency improvement 🤞 
* Refactor batching (behind `enable_v2` feature flag!) to isolate batch writes by data type and sink table

I suspect this will be a quick win prior to isolating the DB or more comprehensive "V2" refactors, so I'm borrowing the new `enable_v2` config flag to condition this for testing in `dev` (then `prod-{us,eu}`) to get it shipped quick.

I propmise I'll return the flag to it's intended use case for the mirror deployment afterwards 👍 

## Does this work well for both Cloud and self-hosted?
Yes

## How did you test this code?
Locally and in CI. Soon in `dev` (with or without mirror deploy)



Diff:
diff --git a/rust/property-defs-rs/.sqlx/query-7fa314878d358c78204cdc1b6282897bcdd5f798d153e263dae6385be787afe4.json b/rust/property-defs-rs/.sqlx/query-7fa314878d358c78204cdc1b6282897bcdd5f798d153e263dae6385be787afe4.json
index 90e3dfb472c0..90ca2967cfbf 100644
--- a/rust/property-defs-rs/.sqlx/query-7fa314878d358c78204cdc1b6282897bcdd5f798d153e263dae6385be787afe4.json
+++ b/rust/property-defs-rs/.sqlx/query-7fa314878d358c78204cdc1b6282897bcdd5f798d153e263dae6385be787afe4.json
@@ -6,7 +6,7 @@
     "parameters": {
       "Left": [
         "Uuid",
-        "Text",
+        "Varchar",
         "Int4",
         "Int8",
         "Timestamptz"
diff --git a/rust/property-defs-rs/.sqlx/query-8c0e2802cd5af5299b4b59860613dd69e107d61cee1a295ca6128d37b74a390f.json b/rust/property-defs-rs/.sqlx/query-8c0e2802cd5af5299b4b59860613dd69e107d61cee1a295ca6128d37b74a390f.json
new file mode 100644
index 000000000000..2a12d67b0484
--- /dev/null
+++ b/rust/property-defs-rs/.sqlx/query-8c0e2802cd5af5299b4b59860613dd69e107d61cee1a295ca6128d37b74a390f.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT count(*) from posthog_propertydefinition WHERE type = 3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "8c0e2802cd5af5299b4b59860613dd69e107d61cee1a295ca6128d37b74a390f"
+}
diff --git a/rust/property-defs-rs/.sqlx/query-af8e999f7d3f77d225953b7e2e7f8c04cc43f014039ea2d05ef42abfc0daa4b8.json b/rust/property-defs-rs/.sqlx/query-af8e999f7d3f77d225953b7e2e7f8c04cc43f014039ea2d05ef42abfc0daa4b8.json
new file mode 100644
index 000000000000..33fbfa77c60d
--- /dev/null
+++ b/rust/property-defs-rs/.sqlx/query-af8e999f7d3f77d225953b7e2e7f8c04cc43f014039ea2d05ef42abfc0daa4b8.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT count(*) from posthog_propertydefinition WHERE type = 2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "af8e999f7d3f77d225953b7e2e7f8c04cc43f014039ea2d05ef42abfc0daa4b8"
+}
diff --git a/rust/property-defs-rs/.sqlx/query-d48ae61a9a754e56c908e98a4e78b5b59ebcfef3c2c2c43d6c54229b312d4dde.json b/rust/property-defs-rs/.sqlx/query-d48ae61a9a754e56c908e98a4e78b5b59ebcfef3c2c2c43d6c54229b312d4dde.json
new file mode 100644
index 000000000000..7f112b80d442
--- /dev/null
+++ b/rust/property-defs-rs/.sqlx/query-d48ae61a9a754e56c908e98a4e78b5b59ebcfef3c2c2c43d6c54229b312d4dde.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT count(*) from posthog_propertydefinition",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "d48ae61a9a754e56c908e98a4e78b5b59ebcfef3c2c2c43d6c54229b312d4dde"
+}
diff --git a/rust/property-defs-rs/.sqlx/query-f7595122b345c487667fb8414e82bc2c251aa5a58eb11fc153e3dac5bb11afc6.json b/rust/property-defs-rs/.sqlx/query-f7595122b345c487667fb8414e82bc2c251aa5a58eb11fc153e3dac5bb11afc6.json
new file mode 100644
index 000000000000..dfcea2f60a6f
--- /dev/null
+++ b/rust/property-defs-rs/.sqlx/query-f7595122b345c487667fb8414e82bc2c251aa5a58eb11fc153e3dac5bb11afc6.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT count(*) from posthog_eventproperty",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f7595122b345c487667fb8414e82bc2c251aa5a58eb11fc153e3dac5bb11afc6"
+}
diff --git a/rust/property-defs-rs/.sqlx/query-fdcd2297f6041a5fe6bc7499a6826db50bd3facc0e40cee8a20c5a070b5723a6.json b/rust/property-defs-rs/.sqlx/query-fdcd2297f6041a5fe6bc7499a6826db50bd3facc0e40cee8a20c5a070b5723a6.json
new file mode 100644
index 000000000000..6811c2bfeb65
--- /dev/null
+++ b/rust/property-defs-rs/.sqlx/query-fdcd2297f6041a5fe6bc7499a6826db50bd3facc0e40cee8a20c5a070b5723a6.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT name from posthog_eventdefinition",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "fdcd2297f6041a5fe6bc7499a6826db50bd3facc0e40cee8a20c5a070b5723a6"
+}
diff --git a/rust/property-defs-rs/src/config.rs b/rust/property-defs-rs/src/config.rs
index 841555f05521..195582906fc7 100644
--- a/rust/property-defs-rs/src/config.rs
+++ b/rust/property-defs-rs/src/config.rs
@@ -115,6 +115,9 @@ pub struct Config {
     // bundled ingest pipeline refactors
     #[envconfig(default = "false")]
     pub enable_v2: bool,
+
+    #[envconfig(default = "100")]
+    pub v2_ingest_batch_size: usize,
 }
 
 #[derive(Clone)]
diff --git a/rust/property-defs-rs/src/lib.rs b/rust/property-defs-rs/src/lib.rs
index 6fe5893d62e7..d8db1299b2b1 100644
--- a/rust/property-defs-rs/src/lib.rs
+++ b/rust/property-defs-rs/src/lib.rs
@@ -1,6 +1,5 @@
 use std::{sync::Arc, time::Duration};
 
-use ahash::AHashSet;
 use app_context::AppContext;
 use common_kafka::kafka_consumer::{RecvErr, SingleTopicConsumer};
 use config::{Config, TeamFilterMode, TeamList};
@@ -11,16 +10,21 @@ use metrics_consts::{
     UPDATES_FILTERED_BY_CACHE, UPDATES_PER_EVENT, UPDATES_SEEN, UPDATE_ISSUE_TIME,
     UPDATE_PRODUCER_OFFSET, WORKER_BLOCKED,
 };
+use types::{Event, Update};
+use v2_batch_ingestion::process_batch_v2;
+
+use ahash::AHashSet;
 use quick_cache::sync::Cache;
+
 use tokio::sync::mpsc::{self, error::TrySendError};
 use tracing::{error, warn};
-use types::{Event, Update};
 
 pub mod api;
 pub mod app_context;
 pub mod config;
 pub mod metrics_consts;
 pub mod types;
+pub mod v2_batch_ingestion;
 
 const BATCH_UPDATE_MAX_ATTEMPTS: u64 = 2;
 const UPDATE_RETRY_DELAY_MS: u64 = 50;
@@ -77,68 +81,84 @@ pub async fn update_consumer_loop(
         let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
         metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
 
-        // We split our update batch into chunks, one per transaction. We know each update touches
-        // exactly one row, so we can issue the chunks in parallel, and smaller batches issue faster,
-        // which helps us with inter-pod deadlocking and retries.
-        let chunk_size = batch.len() / config.max_concurrent_transactions;
-        let mut chunks = vec![Vec::with_capacity(chunk_size); config.max_concurrent_transactions];
-        for (i, update) in batch.drain(..).enumerate() {
-            chunks[i % config.max_concurrent_transactions].push(update);
+        // conditionall enable new write path
+        if config.enable_v2 {
+            process_batch_v2(&config, cache.clone(), &context.pool, batch).await;
+        } else {
+            process_batch_v1(&config, cache.clone(), context.clone(), batch).await;
         }
+    }
+}
 
-        metrics::gauge!(CHUNK_SIZE).set(chunk_size as f64);
-
-        let mut handles = Vec::new();
-        let issue_time = common_metrics::timing_guard(UPDATE_ISSUE_TIME, &[]);
-        for mut chunk in chunks {
-            let m_context = context.clone();
-            let m_cache = cache.clone();
-            let handle = tokio::spawn(async move {
-                let mut tries: u64 = 0;
-                // We occasionally encounter deadlocks while issuing updates, so we retry a few times, and
-                // if we still fail, we drop the batch and clear it's content from the cached update set, because
-                // we assume everything in it will be seen again.
-                while let Err(e) = m_context.issue(&mut chunk, cache_utilization).await {
-                    tries += 1;
-                    if tries > BATCH_UPDATE_MAX_ATTEMPTS {
-                        let chunk_len = chunk.len() as u64;
-                        metrics::counter!(ISSUE_FAILED, &[("reason", "failed")]).increment(1);
-                        metrics::counter!(UPDATES_DROPPED, &[("reason", "batch_write_fail")])
-                            .increment(chunk_len);
-                        error!(
-                            "Issue failed: retries exhausted, dropping batch of size {} with error: {:?}",
-                            chunk_len, e,
-                        );
-                        // We clear any updates that were in this batch from the cache, so that
-                        // if we see them again we'll try again to issue them.
-                        chunk.iter().for_each(|u| {
-                            if m_cache.remove(u).is_some() {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "removed")])
-                                    .increment(1);
-                            } else {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "not_cached")])
-                                    .increment(1);
-                            }
-                        });
-                        return;
-                    }
+async fn process_batch_v1(
+    config: &Config,
+    cache: Arc<Cache<Update, ()>>,
+    context: Arc<AppContext>,
+    mut batch: Vec<Update>,
+) {
+    let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
+    metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
+
+    // We split our update batch into chunks, one per transaction. We know each update touches
+    // exactly one row, so we can issue the chunks in parallel, and smaller batches issue faster,
+    // which helps us with inter-pod deadlocking and retries.
+    let chunk_size = batch.len() / config.max_concurrent_transactions;
+    let mut chunks = vec![Vec::with_capacity(chunk_size); config.max_concurrent_transactions];
+    for (i, update) in batch.drain(..).enumerate() {
+        chunks[i % config.max_concurrent_transactions].push(update);
+    }
+
+    metrics::gauge!(CHUNK_SIZE).set(chunk_size as f64);
 
-                    let jitter = rand::random::<u64>() % 50;
-                    let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
-                    metrics::counter!(ISSUE_FAILED, &[("attempt", format!("retry_{}", tries))])
-                        .increment(1);
-                    warn!("Issue failed: {:?}, sleeping for {}ms", e, delay);
-                    tokio::time::sleep(Duration::from_millis(delay)).await;
+    let mut handles = Vec::new();
+    let issue_time = common_metrics::timing_guard(UPDATE_ISSUE_TIME, &[]);
+    for mut chunk in chunks {
+        let m_context = context.clone();
+        let m_cache = cache.clone();
+        let handle = tokio::spawn(async move {
+            let mut tries: u64 = 0;
+            // We occasionally encounter deadlocks while issuing updates, so we retry a few times, and
+            // if we still fail, we drop the batch and clear it's content from the cached update set, because
+            // we assume everything in it will be seen again.
+            while let Err(e) = m_context.issue(&mut chunk, cache_utilization).await {
+                tries += 1;
+                if tries > BATCH_UPDATE_MAX_ATTEMPTS {
+                    let chunk_len = chunk.len() as u64;
+                    metrics::counter!(ISSUE_FAILED, &[("reason", "failed")]).increment(1);
+                    metrics::counter!(UPDATES_DROPPED, &[("reason", "batch_write_fail")])
+                        .increment(chunk_len);
+                    error!(
+                        "Issue failed: retries exhausted, dropping batch of size {} with error: {:?}",
+                        chunk_len, e,
+                    );
+                    // We clear any updates that were in this batch from the cache, so that
+                    // if we see them again we'll try again to issue them.
+                    chunk.iter().for_each(|u| {
+                        if m_cache.remove(u).is_some() {
+                            metrics::counter!(UPDATES_CACHE, &[("action", "removed")]).increment(1);
+                        } else {
+                            metrics::counter!(UPDATES_CACHE, &[("action", "not_cached")])
+                                .increment(1);
+                        }
+                    });
+                    return;
                 }
-            });
-            handles.push(handle);
-        }
 
-        for handle in handles {
-            handle.await.expect("Issue task failed, exiting");
-        }
-        issue_time.fin();
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
+                metrics::counter!(ISSUE_FAILED, &[("attempt", format!("retry_{}", tries))])
+                    .increment(1);
+                warn!("Issue failed: {:?}, sleeping for {}ms", e, delay);
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+            }
+        });
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.await.expect("Issue task failed, exiting");
     }
+    issue_time.fin();
 }
 
 pub async fn update_producer_loop(
diff --git a/rust/property-defs-rs/src/metrics_consts.rs b/rust/property-defs-rs/src/metrics_consts.rs
index f7d18383d536..78fb77a4c43a 100644
--- a/rust/property-defs-rs/src/metrics_consts.rs
+++ b/rust/property-defs-rs/src/metrics_consts.rs
@@ -27,3 +27,19 @@ pub const ISSUE_FAILED: &str = "prop_defs_issue_failed";
 pub const CHUNK_SIZE: &str = "prop_defs_chunk_size";
 pub const DUPLICATES_IN_BATCH: &str = "prop_defs_duplicates_in_batch";
 pub const SINGLE_UPDATE_ISSUE_TIME: &str = "prop_defs_single_update_issue_time_ms";
+
+//
+// property-defs-rs "v2" (mirror deploy) metric keys below
+//
+
+pub const V2_EVENT_DEFS_BATCH_TIME: &str = "propdefs_v2_eventdefs_batch_ms";
+pub const V2_EVENT_DEFS_BATCH_ATTEMPT: &str = "propdefs_v2_eventdefs_batch_attempt";
+pub const V2_EVENT_DEFS_BATCH_ROWS_AFFECTED: &str = "propdefs_v2_eventdefs_batch_rows";
+
+pub const V2_EVENT_PROPS_BATCH_TIME: &str = "propdefs_v2_eventprops_batch_ms";
+pub const V2_EVENT_PROPS_BATCH_ATTEMPT: &str = "propdefs_v2_eventprops_batch_attempt";
+pub const V2_EVENT_PROPS_BATCH_ROWS_AFFECTED: &str = "propdefs_v2_eventprops_batch_rows";
+
+pub const V2_PROP_DEFS_BATCH_TIME: &str = "propdefs_v2_propdefs_batch_ms";
+pub const V2_PROP_DEFS_BATCH_ATTEMPT: &str = "propdefs_v2_propdefs_batch_attempt";
+pub const V2_PROP_DEFS_BATCH_ROWS_AFFECTED: &str = "propdefs_v2_propdefs_batch_rows";
diff --git a/rust/property-defs-rs/src/v2_batch_ingestion.rs b/rust/property-defs-rs/src/v2_batch_ingestion.rs
new file mode 100644
index 000000000000..a1879552d7eb
--- /dev/null
+++ b/rust/property-defs-rs/src/v2_batch_ingestion.rs
@@ -0,0 +1,418 @@
+use std::{sync::Arc, time::Duration};
+
+use chrono::{DateTime, Utc};
+use quick_cache::sync::Cache;
+use sqlx::PgPool;
+use tokio::task::JoinHandle;
+use tracing::warn;
+use uuid::Uuid;
+
+use crate::{
+    config::Config,
+    metrics_consts::{
+        CACHE_CONSUMED, V2_EVENT_DEFS_BATCH_ATTEMPT, V2_EVENT_DEFS_BATCH_ROWS_AFFECTED,
+        V2_EVENT_DEFS_BATCH_TIME, V2_EVENT_PROPS_BATCH_ATTEMPT, V2_EVENT_PROPS_BATCH_ROWS_AFFECTED,
+        V2_EVENT_PROPS_BATCH_TIME, V2_PROP_DEFS_BATCH_ATTEMPT, V2_PROP_DEFS_BATCH_ROWS_AFFECTED,
+        V2_PROP_DEFS_BATCH_TIME,
+    },
+    types::{EventDefinition, EventProperty, GroupType, PropertyDefinition, Update},
+};
+
+const V2_BATCH_MAX_RETRY_ATTEMPTS: u64 = 3;
+const V2_BATCH_RETRY_DELAY_MS: u64 = 50;
+
+// Derived hash since these are keyed on all fields in the DB
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct EventPropertiesBatch {
+    batch_size: usize,
+    pub team_ids: Vec<i32>,
+    pub project_ids: Vec<i64>,
+    pub event_names: Vec<String>,
+    pub property_names: Vec<String>,
+}
+
+impl EventPropertiesBatch {
+    pub fn new(batch_size: usize) -> Self {
+        Self {
+            batch_size,
+            team_ids: Vec::with_capacity(batch_size),
+            project_ids: Vec::with_capacity(batch_size),
+            event_names: Vec::with_capacity(batch_size),
+            property_names: Vec::with_capacity(batch_size),
+        }
+    }
+
+    pub fn append(&mut self, ep: EventProperty) {
+        self.team_ids.push(ep.team_id);
+        self.project_ids.push(ep.project_id);
+        self.event_names.push(ep.event);
+        self.property_names.push(ep.property);
+    }
+
+    pub fn should_flush_batch(&self) -> bool {
+        self.team_ids.len() >= self.batch_size
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.team_ids.len() == 0
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct EventDefinitionsBatch {
+    batch_size: usize,
+    pub ids: Vec<Uuid>,
+    pub names: Vec<String>,
+    pub team_ids: Vec<i32>,
+    pub project_ids: Vec<i64>,
+    pub last_seen_ats: Vec<DateTime<Utc>>,
+}
+
+impl EventDefinitionsBatch {
+    pub fn new(batch_size: usize) -> Self {
+        Self {
+            batch_size,
+            ids: Vec::with_capacity(batch_size),
+            names: Vec::with_capacity(batch_size),
+            team_ids: Vec::with_capacity(batch_size),
+            project_ids: Vec::with_capacity(batch_size),
+            last_seen_ats: Vec::with_capacity(batch_size),
+        }
+    }
+
+    pub fn append(&mut self, ed: EventDefinition) {
+        self.ids.push(Uuid::now_v7());
+        self.names.push(ed.name);
+        self.team_ids.push(ed.team_id);
+        self.project_ids.push(ed.project_id);
+        self.last_seen_ats.push(ed.last_seen_at);
+    }
+
+    pub fn should_flush_batch(&self) -> bool {
+        self.ids.len() >= self.batch_size
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ids.len() == 0
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct PropertyDefinitionsBatch {
+    batch_size: usize,
+    pub ids: Vec<Uuid>,
+    pub team_ids: Vec<i32>,
+    pub project_ids: Vec<i64>,
+    pub names: Vec<String>,
+    pub are_numerical: Vec<bool>,
+    pub event_types: Vec<i16>,
+    pub property_types: Vec<Option<i16>>,
+    pub group_type_indices: Vec<Option<i16>>,
+    // note: I left off deprecated fields we null out on writes
+}
+
+impl PropertyDefinitionsBatch {
+    pub fn new(batch_size: usize) -> Self {
+        Self {
+            batch_size,
+            ids: Vec::with_capacity(batch_size),
+            team_ids: Vec::with_capacity(batch_size),
+            project_ids: Vec::with_capacity(batch_size),
+            names: Vec::with_capacity(batch_size),
+            are_numerical: Vec::with_capacity(batch_size),
+            property_types: Vec::with_capacity(batch_size),
+            event_types: Vec::with_capacity(batch_size),
+            group_type_indices: Vec::with_capacity(batch_size),
+        }
+    }
+
+    pub fn append(&mut self, pd: PropertyDefinition) {
+        let group_type_index: Option<i16> = match &pd.group_type_index {
+            Some(gt) => match gt {
+                GroupType::Resolved(_, gti) => Some(*gti as i16),
+                GroupType::Unresolved(_) => Some(-1_i16),
+            },
+            _ => Some(-1_i16),
+        };
+        let property_type: Option<i16> = pd.property_type.map(|pt| pt as i16);
+
+        self.ids.push(Uuid::now_v7());
+        self.team_ids.push(pd.team_id);
+        self.project_ids.push(pd.project_id);
+        self.names.push(pd.name);
+        self.are_numerical.push(pd.is_numerical);
+        self.property_types.push(property_type);
+        self.event_types.push(pd.event_type as i16);
+        self.group_type_indices.push(group_type_index);
+    }
+
+    pub fn should_flush_batch(&self) -> bool {
+        self.ids.len() >= self.batch_size
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ids.len() == 0
+    }
+}
+
+// HACK: making this public so the test suite file can live under "../tests/" dir
+pub async fn process_batch_v2(
+    config: &Config,
+    cache: Arc<Cache<Update, ()>>,
+    pool: &PgPool,
+    batch: Vec<Update>,
+) {
+    let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
+    metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
+
+    // TODO(eli): implement v1-style delay while cache is warming?
+
+    // prep reshaped, isolated data batch bufffers and async join handles
+    let mut event_defs = EventDefinitionsBatch::new(config.v2_ingest_batch_size);
+    let mut event_props = EventPropertiesBatch::new(config.v2_ingest_batch_size);
+    let mut prop_defs = PropertyDefinitionsBatch::new(config.v2_ingest_batch_size);
+    let mut handles: Vec<JoinHandle<Result<(), sqlx::Error>>> = vec![];
+
+    // loop on the Update batch, splitting into smaller vectorized PG write batches
+    // and submitted async. note for testing and simplicity, we don't work with
+    // the AppContext in process_batch_v2, just it's PgPool. We clone that all over
+    // the place to pass into `tokio::spawn()` which is fine b/c it's designed for
+    // this and manages concurrent access internaly. Some details here:
+    // https://github.com/launchbadge/sqlx/blob/main/sqlx-core/src/pool/mod.rs#L109-L111
+    for update in batch {
+        match update {
+            Update::Event(ed) => {
+                event_defs.append(ed);
+                if event_defs.should_flush_batch() {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_definitions_batch(event_defs, &pool).await
+                    }));
+                    event_defs = EventDefinitionsBatch::new(config.v2_ingest_batch_size);
+                }
+            }
+            Update::EventProperty(ep) => {
+                event_props.append(ep);
+                if event_props.should_flush_batch() {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_properties_batch(event_props, &pool).await
+                    }));
+                    event_props = EventPropertiesBatch::new(config.v2_ingest_batch_size);
+                }
+            }
+            Update::Property(pd) => {
+                prop_defs.append(pd);
+                if prop_defs.should_flush_batch() {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_property_definitions_batch(prop_defs, &pool).await
+                    }));
+                    prop_defs = PropertyDefinitionsBatch::new(config.v2_ingest_batch_size);
+                }
+            }
+        }
+    }
+
+    // ensure partial batches are flushed to Postgres too
+    if !event_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_definitions_batch(event_defs, &pool).await
+        }));
+    }
+    if !prop_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_property_definitions_batch(prop_defs, &pool).await
+        }));
+    }
+    if !event_props.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_properties_batch(event_props, &pool).await
+        }));
+    }
+
+    for handle in handles {
+        match handle.await {
+            Ok(result) => match result {
+                Ok(_) => continue,
+                Err(db_err) => {
+                    warn!("Batch write exhausted retries: {:?}", db_err);
+                }
+            },
+            Err(join_err) => {
+                warn!("Batch query JoinError: {:?}", join_err);
+            }
+        }
+    }
+}
+
+async fn write_event_properties_batch(
+    batch: EventPropertiesBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_PROPS_BATCH_TIME, &[]);
+    let mut tries = 1;
+
+    loop {
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventproperty (event, property, team_id, project_id)
+                (SELECT * FROM UNNEST(
+                    $1::text[],
+                    $2::text[],
+                    $3::int[],
+                    $4::bigint[])) ON CONFLICT DO NOTHING"#,
+        )
+        .bind(&batch.event_names)
+        .bind(&batch.property_names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .execute(pool)
+        .await;
+
+        match result {
+            Err(e) => {
+                if tries == V2_BATCH_MAX_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * V2_BATCH_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_EVENT_PROPS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
+        }
+    }
+}
+
+async fn write_property_definitions_batch(
+    batch: PropertyDefinitionsBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_PROP_DEFS_BATCH_TIME, &[]);
+    let mut tries: u64 = 1;
+
+    loop {
+        // what if we just ditch properties without a property_type set? why update on conflict at all?
+        let result = sqlx::query(r#"
+            INSERT INTO posthog_propertydefinition (id, name, type, group_type_index, is_numerical, team_id, project_id, property_type)
+                (SELECT * FROM UNNEST(
+                    $1::uuid[],
+                    $2::varchar[],
+                    $3::smallint[],
+                    $4::smallint[],
+                    $5::boolean[],
+                    $6::int[],
+                    $7::bigint[],
+                    $8::varchar[]))
+                ON CONFLICT (
+                    COALESCE(project_id, team_id::bigint), name, type,
+                    COALESCE(group_type_index, -1))
+                DO UPDATE SET property_type=EXCLUDED.property_type
+                WHERE posthog_propertydefinition.property_type IS NULL"#,
+            )
+            .bind(&batch.ids)
+            .bind(&batch.names)
+            .bind(&batch.event_types)
+            .bind(&batch.group_type_indices)
+            .bind(&batch.are_numerical)
+            .bind(&batch.team_ids)
+            .bind(&batch.project_ids)
+            .bind(&batch.property_types)
+            .execute(pool).await;
+
+        match result {
+            Err(e) => {
+                if tries == V2_BATCH_MAX_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * V2_BATCH_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_PROP_DEFS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
+        }
+    }
+}
+
+async fn write_event_definitions_batch(
+    batch: EventDefinitionsBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_DEFS_BATCH_TIME, &[]);
+    let mut tries: u64 = 1;
+
+    loop {
+        // TODO: is last_seen_at critical to the product UX? "ON CONFLICT DO NOTHING" may be much cheaper...
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventdefinition (id, name, team_id, project_id, last_seen_at, created_at)
+                (SELECT * FROM UNNEST (
+                    $1::uuid[],
+                    $2::varchar[],
+                    $3::int[],
+                    $4::bigint[],
+                    $5::timestamptz[],
+                    $5::timestamptz[]))
+                ON CONFLICT (coalesce(project_id, team_id::bigint), name) DO UPDATE
+                    SET last_seen_at=EXCLUDED.last_seen_at
+                    WHERE posthog_eventdefinition.last_seen_at < EXCLUDED.last_seen_at"#,
+        )
+        .bind(&batch.ids)
+        .bind(&batch.names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .bind(&batch.last_seen_ats)
+        .execute(pool)
+        .await;
+
+        match result {
+            Err(e) => {
+                if tries == V2_BATCH_MAX_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_EVENT_DEFS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_EVENT_DEFS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * V2_BATCH_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_EVENT_DEFS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_EVENT_DEFS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
+        }
+    }
+}
diff --git a/rust/property-defs-rs/tests/v2_batch_ingestion.rs b/rust/property-defs-rs/tests/v2_batch_ingestion.rs
new file mode 100644
index 000000000000..ad640fd6a8f8
--- /dev/null
+++ b/rust/property-defs-rs/tests/v2_batch_ingestion.rs
@@ -0,0 +1,203 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use chrono::Utc;
+use quick_cache::sync::Cache;
+use rand::Rng;
+use serde_json::{json, Value};
+use sqlx::PgPool;
+
+use property_defs_rs::{
+    config::Config,
+    types::{Event, PropertyParentType, Update},
+    v2_batch_ingestion::process_batch_v2,
+};
+
+#[sqlx::test(migrations = "./tests/test_migrations")]
+async fn test_simple_batch_write(db: PgPool) {
+    let config = Config::init_with_defaults().unwrap();
+    let cache: Arc<Cache<Update, ()>> = Arc::new(Cache::new(config.cache_capacity));
+    let updates = gen_test_event_updates("$pageview", 100, None);
+    // should decompose into 1 event def, 100 event props, 100 prop defs (of event type)
+    assert_eq!(updates.len(), 201);
+
+    process_batch_v2(&config, cache, &db, updates).await;
+
+    // fetch results and ensure they landed correctly
+    let event_def_name: String = sqlx::query_scalar!(r#"SELECT name from posthog_eventdefinition"#)
+        .fetch_one(&db)
+        .await
+        .unwrap();
+    assert_eq!(String::from("$pageview"), event_def_name);
+
+    let prop_defs_count: Option<i64> =
+        sqlx::query_scalar!(r#"SELECT count(*) from posthog_propertydefinition"#)
+            .fetch_one(&db)
+            .await
+            .unwrap();
+    assert_eq!(Some(100), prop_defs_count);
+
+    let event_props_count: Option<i64> =
+        sqlx::query_scalar!(r#"SELECT count(*) from posthog_eventproperty"#)
+            .fetch_one(&db)
+            .await
+            .unwrap();
+    assert_eq!(Some(100), event_props_count);
+}
+
+#[sqlx::test(migrations = "./tests/test_migrations")]
+async fn test_group_batch_write(db: PgPool) {
+    let config = Config::init_with_defaults().unwrap();
+    let cache: Arc<Cache<Update, ()>> = Arc::new(Cache::new(config.cache_capacity));
+    let updates = gen_test_event_updates("$groupidentify", 100, Some(PropertyParentType::Group));
+    // should decompose into 1 group event def, 100 prop defs (of group type), 100 event props
+    assert_eq!(updates.len(), 201);
+
+    process_batch_v2(&config, cache, &db, updates).await;
+
+    // fetch results and ensure they landed correctly
+    let event_def_name: String = sqlx::query_scalar!(r#"SELECT name from posthog_eventdefinition"#)
+        .fetch_one(&db)
+        .await
+        .unwrap();
+    assert_eq!(String::from("$groupidentify"), event_def_name);
+
+    let prop_defs_count: Option<i64> =
+        sqlx::query_scalar!(r#"SELECT count(*) from posthog_propertydefinition WHERE type = 3"#)
+            .fetch_one(&db)
+            .await
+            .unwrap();
+    assert_eq!(Some(100), prop_defs_count);
+
+    let event_props_count: Option<i64> =
+        sqlx::query_scalar!(r#"SELECT count(*) from posthog_eventproperty"#)
+            .fetch_one(&db)
+            .await
+            .unwrap();
+    assert_eq!(Some(100), event_props_count);
+}
+
+#[sqlx::test(migrations = "./tests/test_migrations")]
+async fn test_person_batch_write(db: PgPool) {
+    let config = Config::init_with_defaults().unwrap();
+    let cache: Arc<Cache<Update, ()>> = Arc::new(Cache::new(config.cache_capacity));
+    let updates =
+        gen_test_event_updates("event_with_person", 100, Some(PropertyParentType::Person));
+    // should decompose into 1 event def, 100 event props, 100 prop defs (50 $set, 50 $set_once props)
+    assert_eq!(updates.len(), 201);
+
+    process_batch_v2(&config, cache, &db, updates).await;
+
+    // fetch results and ensure they landed correctly
+    let event_def_name: String = sqlx::query_scalar!(r#"SELECT name from posthog_eventdefinition"#)
+        .fetch_one(&db)
+        .await
+        .unwrap();
+    assert_eq!(String::from("event_with_person"), event_def_name);
+
+    let prop_defs_count: Option<i64> =
+        sqlx::query_scalar!(r#"SELECT count(*) from posthog_propertydefinition WHERE type = 2"#)
+            .fetch_one(&db)
+            .await
+            .unwrap();
+    assert_eq!(Some(100), prop_defs_count);
+
+    let event_props_count: Option<i64> =
+        sqlx::query_scalar!(r#"SELECT count(*) from posthog_eventproperty"#)
+            .fetch_one(&db)
+            .await
+            .unwrap();
+    assert_eq!(Some(100), event_props_count);
+}
+
+fn gen_test_event_updates(
+    event_name: &'static str,
+    num_props: usize,
+    prop_type: Option<PropertyParentType>,
+) -> Vec<Update> {
+    let prop_kvs = (0..num_props)
+        .map(gen_test_prop_key_value)
+        .collect::<Vec<(String, Value)>>();
+
+    // let's be *just* realistic enough to exercise we're decomposing the Event properly in unit tests
+    let mut properties = HashMap::<String, Value>::new();
+    match prop_type {
+        Some(PropertyParentType::Person) => {
+            let mut set_props = HashMap::<String, Value>::new();
+            let mut set_once_props = HashMap::<String, Value>::new();
+
+            // split the total props between $set and $set_once buckets.
+            // these will be flattened back into a single unique group of props
+            // during parsing into Update records
+            for kv in prop_kvs.iter().enumerate() {
+                if kv.0 % 2 == 0 {
+                    set_props.insert(kv.1 .0.clone(), kv.1 .1.clone());
+                } else {
+                    set_once_props.insert(kv.1 .0.clone(), kv.1 .1.clone());
+                }
+            }
+            properties.insert(String::from("$set"), json!(set_props));
+            properties.insert(String::from("$set_once"), json!(set_once_props));
+        }
+        Some(PropertyParentType::Group) => {
+            let mut group_props = HashMap::<String, Value>::new();
+            for kv in prop_kvs {
+                group_props.insert(kv.0, kv.1);
+            }
+            properties.insert(
+                String::from("$group_type"),
+                Value::String(String::from("Organization")),
+            );
+            properties.insert(String::from("$group_set"), json!(group_props));
+        }
+        _ => {
+            // generate flat list of "vanilla" event props
+            for kv in prop_kvs {
+                properties.insert(kv.0, kv.1);
+            }
+        }
+    }
+
+    let properties = json!(properties).to_string();
+
+    let event = json!({
+        "team_id": 111,
+        "project_id": 111,
+        "event": event_name,
+        "properties": properties,
+    });
+
+    serde_json::from_value::<Event>(event)
+        .unwrap()
+        .into_updates(10000)
+}
+
+fn gen_test_prop_key_value(ndx: usize) -> (String, Value) {
+    use rand::seq::SliceRandom;
+
+    match [0, 1, 2, 3].choose(&mut rand::thread_rng()) {
+        // String
+        Some(&0) => (
+            format!("str_prop_{}", ndx),
+            Value::String(format!("value_{}", ndx)),
+        ),
+
+        // Boolean
+        Some(&1) => (format!("bool_prop_{}", ndx), Value::Bool(true)),
+
+        // Numeric
+        Some(&2) => (
+            format!("numeric_prop_{}", ndx),
+            json!(rand::thread_rng().gen_range(0..1000)),
+        ),
+
+        // DateTime
+        Some(&3) => (
+            format!("datetime_prop_{}", ndx),
+            Value::String(Utc::now().to_rfc3339()),
+        ),
+
+        // skipping Duration as it's unused for now
+        _ => panic!("not reachable"),
+    }
+}


Reviews:
user: login: greptile-apps[bot]
body: 
## PR Summary

The PR refactors the batching process by vectorizing writes for property definitions, isolating updates per data type with new batch structures, and expanding performance metrics under the enable_v2 flag.  
- Expanded metrics constants in /rust/property-defs-rs/src/metrics_consts.rs for v2 batch performance tracking.  
- Added tests in /rust/property-defs-rs/tests/v2_batch_ingestion.rs to validate proper batching behavior for event, group, and person updates.  
- Modified batch structures in /rust/property-defs-rs/src/types.rs to preallocate vectors and signal full batches.  
- In /rust/property-defs-rs/src/lib.rs, identified missing await for tokio::time::sleep in retry loops.

<sub>4 file(s) reviewed, 1 comment(s)</sub>
<sub>[Edit PR Review Bot Settings](https://app.greptile.com/review/github) | [Greptile](https://greptile.com?utm_source=greptile_expert&utm_medium=github&utm_campaign=code_reviews&utm_content=posthog_30425)</sub>
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T20:34:54Z
user: login: pl
body: LGTM, let's merge this. A couple of small comments, if you think they make sense and can make the changes quickly, would be nice to make the updates, but they're just nitpicks. Feel free to merge either way.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T16:00:01Z
user: login: eli-r-ph
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T16:22:28Z
user: login: eli-r-ph
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T17:38:11Z
user: login: eli-r-ph
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T17:38:33Z
user: login: eli-r-ph
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T17:39:10Z

Review comments:
diff_hunk: @@ -77,70 +84,343 @@ pub async fn update_consumer_loop(
         let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
         metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
 
-        // We split our update batch into chunks, one per transaction. We know each update touches
-        // exactly one row, so we can issue the chunks in parallel, and smaller batches issue faster,
-        // which helps us with inter-pod deadlocking and retries.
-        let chunk_size = batch.len() / config.max_concurrent_transactions;
-        let mut chunks = vec![Vec::with_capacity(chunk_size); config.max_concurrent_transactions];
-        for (i, update) in batch.drain(..).enumerate() {
-            chunks[i % config.max_concurrent_transactions].push(update);
+        // conditionall enable new write path
+        if config.enable_v2 {
+            process_batch_v2(&config, cache.clone(), &context.pool, batch).await;
+        } else {
+            process_batch_v1(&config, cache.clone(), context.clone(), batch).await;
         }
+    }
+}
 
-        metrics::gauge!(CHUNK_SIZE).set(chunk_size as f64);
-
-        let mut handles = Vec::new();
-        let issue_time = common_metrics::timing_guard(UPDATE_ISSUE_TIME, &[]);
-        for mut chunk in chunks {
-            let m_context = context.clone();
-            let m_cache = cache.clone();
-            let handle = tokio::spawn(async move {
-                let mut tries: u64 = 0;
-                // We occasionally encounter deadlocks while issuing updates, so we retry a few times, and
-                // if we still fail, we drop the batch and clear it's content from the cached update set, because
-                // we assume everything in it will be seen again.
-                while let Err(e) = m_context.issue(&mut chunk, cache_utilization).await {
-                    tries += 1;
-                    if tries > BATCH_UPDATE_MAX_ATTEMPTS {
-                        let chunk_len = chunk.len() as u64;
-                        metrics::counter!(ISSUE_FAILED, &[("reason", "failed")]).increment(1);
-                        metrics::counter!(UPDATES_DROPPED, &[("reason", "batch_write_fail")])
-                            .increment(chunk_len);
-                        error!(
-                            "Issue failed: retries exhausted, dropping batch of size {} with error: {:?}",
-                            chunk_len, e,
-                        );
-                        // We clear any updates that were in this batch from the cache, so that
-                        // if we see them again we'll try again to issue them.
-                        chunk.iter().for_each(|u| {
-                            if m_cache.remove(u).is_some() {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "removed")])
-                                    .increment(1);
-                            } else {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "not_cached")])
-                                    .increment(1);
-                            }
-                        });
-                        return;
-                    }
+// HACK: making this public so the test suite file can live under "../tests/" dir
+pub async fn process_batch_v2(
+    config: &Config,
+    cache: Arc<Cache<Update, ()>>,
+    pool: &PgPool,
+    batch: Vec<Update>,
+) {
+    let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
+    metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
+
+    // prep reshaped, isolated data batch bufffers and async join handles
+    let mut event_defs = EventDefinitionsBatch::new();
+    let mut event_props = EventPropertiesBatch::new();
+    let mut prop_defs = PropertyDefinitionsBatch::new();
+    let mut handles: Vec<JoinHandle<Result<(), sqlx::Error>>> = vec![];
+
+    // loop on the Update batch, splitting into smaller vectorized PG write batches
+    // and submitted async. note for testing and simplicity, we don't work with
+    // the AppContext in process_batch_v2, just it's PgPool. We clone that all over
+    // the place to pass into `tokio::spawn()` which is fine b/c it's designed for
+    // this and manages concurrent access internaly. Some details here:
+    // https://github.com/launchbadge/sqlx/blob/main/sqlx-core/src/pool/mod.rs#L109-L111
+    for update in batch {
+        match update {
+            Update::Event(ed) => {
+                if event_defs.append(ed) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_definitions_batch(event_defs, &pool).await
+                    }));
+                    event_defs = EventDefinitionsBatch::new();
+                }
+            }
+            Update::EventProperty(ep) => {
+                if event_props.append(ep) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_properties_batch(event_props, &pool).await
+                    }));
+                    event_props = EventPropertiesBatch::new();
+                }
+            }
+            Update::Property(pd) => {
+                if prop_defs.append(pd) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_property_definitions_batch(prop_defs, &pool).await
+                    }));
+                    prop_defs = PropertyDefinitionsBatch::new();
+                }
+            }
+        }
+    }
 
-                    let jitter = rand::random::<u64>() % 50;
-                    let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
-                    metrics::counter!(ISSUE_FAILED, &[("attempt", format!("retry_{}", tries))])
-                        .increment(1);
-                    warn!("Issue failed: {:?}, sleeping for {}ms", e, delay);
-                    tokio::time::sleep(Duration::from_millis(delay)).await;
+    // ensure partial batches are flushed to Postgres too
+    if !event_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_definitions_batch(event_defs, &pool).await
+        }));
+    }
+    if !prop_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_property_definitions_batch(prop_defs, &pool).await
+        }));
+    }
+    if !event_props.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_properties_batch(event_props, &pool).await
+        }));
+    }
+
+    for handle in handles {
+        match handle.await {
+            Ok(result) => match result {
+                Ok(_) => continue,
+                Err(db_err) => {
+                    warn!("Batch write exhausted retries: {:?}", db_err);
                 }
-            });
-            handles.push(handle);
+            },
+            Err(join_err) => {
+                warn!("Batch query JoinError: {:?}", join_err);
+            }
         }
+    }
+}
+
+async fn write_event_properties_batch(
+    batch: EventPropertiesBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_PROPS_BATCH_TIME, &[]);
+    let mut tries = 1;
+
+    loop {
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventproperty (event, property, team_id, project_id)
+                (SELECT * FROM UNNEST(
+                    $1::text[],
+                    $2::text[],
+                    $3::int[],
+                    $4::bigint[])) ON CONFLICT DO NOTHING"#,
+        )
+        .bind(&batch.event_names)
+        .bind(&batch.property_names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .execute(pool)
+        .await;
 
-        for handle in handles {
-            handle.await.expect("Issue task failed, exiting");
+        match result {
+            Err(e) => {
+                if tries == MAX_V2_BATCH_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
+                let _unused = tokio::time::sleep(Duration::from_millis(delay));
+                tries += 1;
+            }
path: rust/property-defs-rs/src/lib.rs
user: login: greptile-apps[bot]
body: logic: Missing '.await' on tokio::time::sleep in retry loop; add .await to delay retries properly.

```suggestion
                let jitter = rand::random::<u64>() % 50;
                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
                tokio::time::sleep(Duration::from_millis(delay)).await;
                tries += 1;
            }
```
created_at: 2025-03-27T20:34:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -77,70 +84,343 @@ pub async fn update_consumer_loop(
         let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
         metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
 
-        // We split our update batch into chunks, one per transaction. We know each update touches
-        // exactly one row, so we can issue the chunks in parallel, and smaller batches issue faster,
-        // which helps us with inter-pod deadlocking and retries.
-        let chunk_size = batch.len() / config.max_concurrent_transactions;
-        let mut chunks = vec![Vec::with_capacity(chunk_size); config.max_concurrent_transactions];
-        for (i, update) in batch.drain(..).enumerate() {
-            chunks[i % config.max_concurrent_transactions].push(update);
+        // conditionall enable new write path
+        if config.enable_v2 {
+            process_batch_v2(&config, cache.clone(), &context.pool, batch).await;
+        } else {
+            process_batch_v1(&config, cache.clone(), context.clone(), batch).await;
         }
+    }
+}
 
-        metrics::gauge!(CHUNK_SIZE).set(chunk_size as f64);
-
-        let mut handles = Vec::new();
-        let issue_time = common_metrics::timing_guard(UPDATE_ISSUE_TIME, &[]);
-        for mut chunk in chunks {
-            let m_context = context.clone();
-            let m_cache = cache.clone();
-            let handle = tokio::spawn(async move {
-                let mut tries: u64 = 0;
-                // We occasionally encounter deadlocks while issuing updates, so we retry a few times, and
-                // if we still fail, we drop the batch and clear it's content from the cached update set, because
-                // we assume everything in it will be seen again.
-                while let Err(e) = m_context.issue(&mut chunk, cache_utilization).await {
-                    tries += 1;
-                    if tries > BATCH_UPDATE_MAX_ATTEMPTS {
-                        let chunk_len = chunk.len() as u64;
-                        metrics::counter!(ISSUE_FAILED, &[("reason", "failed")]).increment(1);
-                        metrics::counter!(UPDATES_DROPPED, &[("reason", "batch_write_fail")])
-                            .increment(chunk_len);
-                        error!(
-                            "Issue failed: retries exhausted, dropping batch of size {} with error: {:?}",
-                            chunk_len, e,
-                        );
-                        // We clear any updates that were in this batch from the cache, so that
-                        // if we see them again we'll try again to issue them.
-                        chunk.iter().for_each(|u| {
-                            if m_cache.remove(u).is_some() {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "removed")])
-                                    .increment(1);
-                            } else {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "not_cached")])
-                                    .increment(1);
-                            }
-                        });
-                        return;
-                    }
+// HACK: making this public so the test suite file can live under "../tests/" dir
+pub async fn process_batch_v2(
+    config: &Config,
+    cache: Arc<Cache<Update, ()>>,
+    pool: &PgPool,
+    batch: Vec<Update>,
+) {
+    let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
+    metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
+
+    // prep reshaped, isolated data batch bufffers and async join handles
+    let mut event_defs = EventDefinitionsBatch::new();
+    let mut event_props = EventPropertiesBatch::new();
+    let mut prop_defs = PropertyDefinitionsBatch::new();
+    let mut handles: Vec<JoinHandle<Result<(), sqlx::Error>>> = vec![];
+
+    // loop on the Update batch, splitting into smaller vectorized PG write batches
+    // and submitted async. note for testing and simplicity, we don't work with
+    // the AppContext in process_batch_v2, just it's PgPool. We clone that all over
+    // the place to pass into `tokio::spawn()` which is fine b/c it's designed for
+    // this and manages concurrent access internaly. Some details here:
+    // https://github.com/launchbadge/sqlx/blob/main/sqlx-core/src/pool/mod.rs#L109-L111
+    for update in batch {
+        match update {
+            Update::Event(ed) => {
+                if event_defs.append(ed) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_definitions_batch(event_defs, &pool).await
+                    }));
+                    event_defs = EventDefinitionsBatch::new();
+                }
+            }
+            Update::EventProperty(ep) => {
+                if event_props.append(ep) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_properties_batch(event_props, &pool).await
+                    }));
+                    event_props = EventPropertiesBatch::new();
+                }
+            }
+            Update::Property(pd) => {
+                if prop_defs.append(pd) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_property_definitions_batch(prop_defs, &pool).await
+                    }));
+                    prop_defs = PropertyDefinitionsBatch::new();
+                }
+            }
+        }
+    }
 
-                    let jitter = rand::random::<u64>() % 50;
-                    let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
-                    metrics::counter!(ISSUE_FAILED, &[("attempt", format!("retry_{}", tries))])
-                        .increment(1);
-                    warn!("Issue failed: {:?}, sleeping for {}ms", e, delay);
-                    tokio::time::sleep(Duration::from_millis(delay)).await;
+    // ensure partial batches are flushed to Postgres too
+    if !event_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_definitions_batch(event_defs, &pool).await
+        }));
+    }
+    if !prop_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_property_definitions_batch(prop_defs, &pool).await
+        }));
+    }
+    if !event_props.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_properties_batch(event_props, &pool).await
+        }));
+    }
+
+    for handle in handles {
+        match handle.await {
+            Ok(result) => match result {
+                Ok(_) => continue,
+                Err(db_err) => {
+                    warn!("Batch write exhausted retries: {:?}", db_err);
                 }
-            });
-            handles.push(handle);
+            },
+            Err(join_err) => {
+                warn!("Batch query JoinError: {:?}", join_err);
+            }
         }
+    }
+}
+
+async fn write_event_properties_batch(
+    batch: EventPropertiesBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_PROPS_BATCH_TIME, &[]);
+    let mut tries = 1;
+
+    loop {
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventproperty (event, property, team_id, project_id)
+                (SELECT * FROM UNNEST(
+                    $1::text[],
+                    $2::text[],
+                    $3::int[],
+                    $4::bigint[])) ON CONFLICT DO NOTHING"#,
+        )
+        .bind(&batch.event_names)
+        .bind(&batch.property_names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .execute(pool)
+        .await;
 
-        for handle in handles {
-            handle.await.expect("Issue task failed, exiting");
+        match result {
+            Err(e) => {
+                if tries == MAX_V2_BATCH_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_EVENT_PROPS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
         }
-        issue_time.fin();
     }
 }
 
+async fn write_property_definitions_batch(
+    batch: PropertyDefinitionsBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_PROP_DEFS_BATCH_TIME, &[]);
+    let mut tries: u64 = 1;
+
+    loop {
+        // what if we just ditch properties without a property_type set? why update on conflict at all?
+        let result = sqlx::query(r#"
+            INSERT INTO posthog_propertydefinition (id, name, type, group_type_index, is_numerical, team_id, project_id, property_type)
+                (SELECT * FROM UNNEST(
+                    $1::uuid[],
+                    $2::varchar[],
+                    $3::smallint[],
+                    $4::smallint[],
+                    $5::boolean[],
+                    $6::int[],
+                    $7::bigint[],
+                    $8::varchar[]))
+                ON CONFLICT (
+                    COALESCE(project_id, team_id::bigint), name, type,
+                    COALESCE(group_type_index, -1))
+                DO UPDATE SET property_type=EXCLUDED.property_type
+                WHERE posthog_propertydefinition.property_type IS NULL"#,
+            )
+            .bind(&batch.ids)
+            .bind(&batch.names)
+            .bind(&batch.event_types)
+            .bind(&batch.group_type_indices)
+            .bind(&batch.are_numerical)
+            .bind(&batch.team_ids)
+            .bind(&batch.project_ids)
+            .bind(&batch.property_types)
+            .execute(pool).await;
+
+        match result {
+            Err(e) => {
+                if tries == MAX_V2_BATCH_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_PROP_DEFS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
+        }
+    }
+}
+
+async fn write_event_definitions_batch(
+    batch: EventDefinitionsBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_DEFS_BATCH_TIME, &[]);
+    let mut tries: u64 = 1;
+
+    loop {
+        // TODO: is last_seen_at critical to the product UX? "ON CONFLICT DO NOTHING" may be much cheaper...
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventdefinition (id, name, team_id, project_id, last_seen_at, created_at)
+                (SELECT * FROM UNNEST (
+                    $1::uuid[],
+                    $2::varchar[],
+                    $3::int[],
+                    $4::bigint[],
+                    $5::timestamptz[],
+                    $5::timestamptz[]))
+                ON CONFLICT (coalesce(project_id, team_id::bigint), name) DO UPDATE
+                    SET last_seen_at=EXCLUDED.last_seen_at
+                    WHERE posthog_eventdefinition.last_seen_at < EXCLUDED.last_seen_at"#,
+        )
+        .bind(&batch.ids)
+        .bind(&batch.names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .bind(&batch.last_seen_ats)
+        .execute(pool)
+        .await;
+
+        match result {
path: rust/property-defs-rs/src/lib.rs
user: login: pl
body: nit(non-blocking): looks like this error handling loop could be broken out into a separate function.
created_at: 2025-03-28T15:53:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -120,6 +122,66 @@ pub struct PropertyDefinition {
     pub query_usage_30_day: Option<i64>,      // Deprecated
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
path: rust/property-defs-rs/src/types.rs
user: login: pl
body: nit: would be great to put those new structs in a separate file – I'm not a huge fan of those generic "types" modules.
created_at: 2025-03-28T15:54:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -120,6 +122,66 @@ pub struct PropertyDefinition {
     pub query_usage_30_day: Option<i64>,      // Deprecated
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct PropertyDefinitionsBatch {
+    pub ids: Vec<Uuid>,
+    pub team_ids: Vec<i32>,
+    pub project_ids: Vec<i64>,
+    pub names: Vec<String>,
+    pub are_numerical: Vec<bool>,
+    pub event_types: Vec<i16>,
+    pub property_types: Vec<Option<i16>>,
+    pub group_type_indices: Vec<Option<i16>>,
+    // note: I left off deprecated fields we null out on writes
+}
+
+impl Default for PropertyDefinitionsBatch {
+    fn default() -> Self {
+        Self {
+            ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            team_ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            project_ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            names: Vec::with_capacity(WRITE_BATCH_SIZE),
+            are_numerical: Vec::with_capacity(WRITE_BATCH_SIZE),
+            property_types: Vec::with_capacity(WRITE_BATCH_SIZE),
+            event_types: Vec::with_capacity(WRITE_BATCH_SIZE),
+            group_type_indices: Vec::with_capacity(WRITE_BATCH_SIZE),
+        }
+    }
+}
+
+impl PropertyDefinitionsBatch {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn append(&mut self, pd: PropertyDefinition) -> bool {
+        let group_type_index: Option<i16> = match &pd.group_type_index {
+            Some(gt) => match gt {
+                GroupType::Resolved(_, gti) => Some(*gti as i16),
+                GroupType::Unresolved(_) => Some(-1_i16),
+            },
+            _ => Some(-1_i16),
+        };
+        let property_type: Option<i16> = pd.property_type.map(|pt| pt as i16);
+
+        self.ids.push(Uuid::now_v7());
+        self.team_ids.push(pd.team_id);
+        self.project_ids.push(pd.project_id);
+        self.names.push(pd.name);
+        self.are_numerical.push(pd.is_numerical);
+        self.property_types.push(property_type);
+        self.event_types.push(pd.event_type as i16);
+        self.group_type_indices.push(group_type_index);
+
+        self.ids.len() >= WRITE_BATCH_SIZE
path: rust/property-defs-rs/src/types.rs
user: login: pl
body: nit: I was a little bit confused when reading the code that appends updates to those batches. It's not clear from the interface nor the name that this struct has limits and indicates whether a batch should be flushed. Might be more readable to have a more explicit `should_flush` method.
created_at: 2025-03-28T15:56:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -120,6 +122,66 @@ pub struct PropertyDefinition {
     pub query_usage_30_day: Option<i64>,      // Deprecated
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct PropertyDefinitionsBatch {
+    pub ids: Vec<Uuid>,
+    pub team_ids: Vec<i32>,
+    pub project_ids: Vec<i64>,
+    pub names: Vec<String>,
+    pub are_numerical: Vec<bool>,
+    pub event_types: Vec<i16>,
+    pub property_types: Vec<Option<i16>>,
+    pub group_type_indices: Vec<Option<i16>>,
+    // note: I left off deprecated fields we null out on writes
+}
+
+impl Default for PropertyDefinitionsBatch {
+    fn default() -> Self {
+        Self {
+            ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            team_ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            project_ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            names: Vec::with_capacity(WRITE_BATCH_SIZE),
+            are_numerical: Vec::with_capacity(WRITE_BATCH_SIZE),
+            property_types: Vec::with_capacity(WRITE_BATCH_SIZE),
+            event_types: Vec::with_capacity(WRITE_BATCH_SIZE),
+            group_type_indices: Vec::with_capacity(WRITE_BATCH_SIZE),
+        }
+    }
+}
+
+impl PropertyDefinitionsBatch {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn append(&mut self, pd: PropertyDefinition) -> bool {
+        let group_type_index: Option<i16> = match &pd.group_type_index {
+            Some(gt) => match gt {
+                GroupType::Resolved(_, gti) => Some(*gti as i16),
+                GroupType::Unresolved(_) => Some(-1_i16),
+            },
+            _ => Some(-1_i16),
+        };
+        let property_type: Option<i16> = pd.property_type.map(|pt| pt as i16);
+
+        self.ids.push(Uuid::now_v7());
+        self.team_ids.push(pd.team_id);
+        self.project_ids.push(pd.project_id);
+        self.names.push(pd.name);
+        self.are_numerical.push(pd.is_numerical);
+        self.property_types.push(property_type);
+        self.event_types.push(pd.event_type as i16);
+        self.group_type_indices.push(group_type_index);
+
+        self.ids.len() >= WRITE_BATCH_SIZE
path: rust/property-defs-rs/src/types.rs
user: login: eli-r-ph
body: Nice, will do.

I'm still on the fence about hardcoding the batch size, even at this stage too - might move that to a config var too 🤔 
created_at: 2025-03-28T16:22:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -77,70 +84,343 @@ pub async fn update_consumer_loop(
         let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
         metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
 
-        // We split our update batch into chunks, one per transaction. We know each update touches
-        // exactly one row, so we can issue the chunks in parallel, and smaller batches issue faster,
-        // which helps us with inter-pod deadlocking and retries.
-        let chunk_size = batch.len() / config.max_concurrent_transactions;
-        let mut chunks = vec![Vec::with_capacity(chunk_size); config.max_concurrent_transactions];
-        for (i, update) in batch.drain(..).enumerate() {
-            chunks[i % config.max_concurrent_transactions].push(update);
+        // conditionall enable new write path
+        if config.enable_v2 {
+            process_batch_v2(&config, cache.clone(), &context.pool, batch).await;
+        } else {
+            process_batch_v1(&config, cache.clone(), context.clone(), batch).await;
         }
+    }
+}
 
-        metrics::gauge!(CHUNK_SIZE).set(chunk_size as f64);
-
-        let mut handles = Vec::new();
-        let issue_time = common_metrics::timing_guard(UPDATE_ISSUE_TIME, &[]);
-        for mut chunk in chunks {
-            let m_context = context.clone();
-            let m_cache = cache.clone();
-            let handle = tokio::spawn(async move {
-                let mut tries: u64 = 0;
-                // We occasionally encounter deadlocks while issuing updates, so we retry a few times, and
-                // if we still fail, we drop the batch and clear it's content from the cached update set, because
-                // we assume everything in it will be seen again.
-                while let Err(e) = m_context.issue(&mut chunk, cache_utilization).await {
-                    tries += 1;
-                    if tries > BATCH_UPDATE_MAX_ATTEMPTS {
-                        let chunk_len = chunk.len() as u64;
-                        metrics::counter!(ISSUE_FAILED, &[("reason", "failed")]).increment(1);
-                        metrics::counter!(UPDATES_DROPPED, &[("reason", "batch_write_fail")])
-                            .increment(chunk_len);
-                        error!(
-                            "Issue failed: retries exhausted, dropping batch of size {} with error: {:?}",
-                            chunk_len, e,
-                        );
-                        // We clear any updates that were in this batch from the cache, so that
-                        // if we see them again we'll try again to issue them.
-                        chunk.iter().for_each(|u| {
-                            if m_cache.remove(u).is_some() {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "removed")])
-                                    .increment(1);
-                            } else {
-                                metrics::counter!(UPDATES_CACHE, &[("action", "not_cached")])
-                                    .increment(1);
-                            }
-                        });
-                        return;
-                    }
+// HACK: making this public so the test suite file can live under "../tests/" dir
+pub async fn process_batch_v2(
+    config: &Config,
+    cache: Arc<Cache<Update, ()>>,
+    pool: &PgPool,
+    batch: Vec<Update>,
+) {
+    let cache_utilization = cache.len() as f64 / config.cache_capacity as f64;
+    metrics::gauge!(CACHE_CONSUMED).set(cache_utilization);
+
+    // prep reshaped, isolated data batch bufffers and async join handles
+    let mut event_defs = EventDefinitionsBatch::new();
+    let mut event_props = EventPropertiesBatch::new();
+    let mut prop_defs = PropertyDefinitionsBatch::new();
+    let mut handles: Vec<JoinHandle<Result<(), sqlx::Error>>> = vec![];
+
+    // loop on the Update batch, splitting into smaller vectorized PG write batches
+    // and submitted async. note for testing and simplicity, we don't work with
+    // the AppContext in process_batch_v2, just it's PgPool. We clone that all over
+    // the place to pass into `tokio::spawn()` which is fine b/c it's designed for
+    // this and manages concurrent access internaly. Some details here:
+    // https://github.com/launchbadge/sqlx/blob/main/sqlx-core/src/pool/mod.rs#L109-L111
+    for update in batch {
+        match update {
+            Update::Event(ed) => {
+                if event_defs.append(ed) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_definitions_batch(event_defs, &pool).await
+                    }));
+                    event_defs = EventDefinitionsBatch::new();
+                }
+            }
+            Update::EventProperty(ep) => {
+                if event_props.append(ep) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_event_properties_batch(event_props, &pool).await
+                    }));
+                    event_props = EventPropertiesBatch::new();
+                }
+            }
+            Update::Property(pd) => {
+                if prop_defs.append(pd) {
+                    let pool = pool.clone();
+                    handles.push(tokio::spawn(async move {
+                        write_property_definitions_batch(prop_defs, &pool).await
+                    }));
+                    prop_defs = PropertyDefinitionsBatch::new();
+                }
+            }
+        }
+    }
 
-                    let jitter = rand::random::<u64>() % 50;
-                    let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
-                    metrics::counter!(ISSUE_FAILED, &[("attempt", format!("retry_{}", tries))])
-                        .increment(1);
-                    warn!("Issue failed: {:?}, sleeping for {}ms", e, delay);
-                    tokio::time::sleep(Duration::from_millis(delay)).await;
+    // ensure partial batches are flushed to Postgres too
+    if !event_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_definitions_batch(event_defs, &pool).await
+        }));
+    }
+    if !prop_defs.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_property_definitions_batch(prop_defs, &pool).await
+        }));
+    }
+    if !event_props.is_empty() {
+        let pool = pool.clone();
+        handles.push(tokio::spawn(async move {
+            write_event_properties_batch(event_props, &pool).await
+        }));
+    }
+
+    for handle in handles {
+        match handle.await {
+            Ok(result) => match result {
+                Ok(_) => continue,
+                Err(db_err) => {
+                    warn!("Batch write exhausted retries: {:?}", db_err);
                 }
-            });
-            handles.push(handle);
+            },
+            Err(join_err) => {
+                warn!("Batch query JoinError: {:?}", join_err);
+            }
         }
+    }
+}
+
+async fn write_event_properties_batch(
+    batch: EventPropertiesBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_PROPS_BATCH_TIME, &[]);
+    let mut tries = 1;
+
+    loop {
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventproperty (event, property, team_id, project_id)
+                (SELECT * FROM UNNEST(
+                    $1::text[],
+                    $2::text[],
+                    $3::int[],
+                    $4::bigint[])) ON CONFLICT DO NOTHING"#,
+        )
+        .bind(&batch.event_names)
+        .bind(&batch.property_names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .execute(pool)
+        .await;
 
-        for handle in handles {
-            handle.await.expect("Issue task failed, exiting");
+        match result {
+            Err(e) => {
+                if tries == MAX_V2_BATCH_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_EVENT_PROPS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_EVENT_PROPS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
         }
-        issue_time.fin();
     }
 }
 
+async fn write_property_definitions_batch(
+    batch: PropertyDefinitionsBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_PROP_DEFS_BATCH_TIME, &[]);
+    let mut tries: u64 = 1;
+
+    loop {
+        // what if we just ditch properties without a property_type set? why update on conflict at all?
+        let result = sqlx::query(r#"
+            INSERT INTO posthog_propertydefinition (id, name, type, group_type_index, is_numerical, team_id, project_id, property_type)
+                (SELECT * FROM UNNEST(
+                    $1::uuid[],
+                    $2::varchar[],
+                    $3::smallint[],
+                    $4::smallint[],
+                    $5::boolean[],
+                    $6::int[],
+                    $7::bigint[],
+                    $8::varchar[]))
+                ON CONFLICT (
+                    COALESCE(project_id, team_id::bigint), name, type,
+                    COALESCE(group_type_index, -1))
+                DO UPDATE SET property_type=EXCLUDED.property_type
+                WHERE posthog_propertydefinition.property_type IS NULL"#,
+            )
+            .bind(&batch.ids)
+            .bind(&batch.names)
+            .bind(&batch.event_types)
+            .bind(&batch.group_type_indices)
+            .bind(&batch.are_numerical)
+            .bind(&batch.team_ids)
+            .bind(&batch.project_ids)
+            .bind(&batch.property_types)
+            .execute(pool).await;
+
+        match result {
+            Err(e) => {
+                if tries == MAX_V2_BATCH_RETRY_ATTEMPTS {
+                    metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "failed")]);
+                    total_time.fin();
+                    return Err(e);
+                }
+
+                metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "retry")]);
+                let jitter = rand::random::<u64>() % 50;
+                let delay: u64 = tries * UPDATE_RETRY_DELAY_MS + jitter;
+                tokio::time::sleep(Duration::from_millis(delay)).await;
+                tries += 1;
+            }
+
+            Ok(pgq_result) => {
+                let count = pgq_result.rows_affected();
+                metrics::counter!(V2_PROP_DEFS_BATCH_ATTEMPT, &[("result", "success")]);
+                common_metrics::inc(V2_PROP_DEFS_BATCH_ROWS_AFFECTED, &[], count);
+                total_time.fin();
+                return Ok(());
+            }
+        }
+    }
+}
+
+async fn write_event_definitions_batch(
+    batch: EventDefinitionsBatch,
+    pool: &PgPool,
+) -> Result<(), sqlx::Error> {
+    let total_time = common_metrics::timing_guard(V2_EVENT_DEFS_BATCH_TIME, &[]);
+    let mut tries: u64 = 1;
+
+    loop {
+        // TODO: is last_seen_at critical to the product UX? "ON CONFLICT DO NOTHING" may be much cheaper...
+        let result = sqlx::query(
+            r#"
+            INSERT INTO posthog_eventdefinition (id, name, team_id, project_id, last_seen_at, created_at)
+                (SELECT * FROM UNNEST (
+                    $1::uuid[],
+                    $2::varchar[],
+                    $3::int[],
+                    $4::bigint[],
+                    $5::timestamptz[],
+                    $5::timestamptz[]))
+                ON CONFLICT (coalesce(project_id, team_id::bigint), name) DO UPDATE
+                    SET last_seen_at=EXCLUDED.last_seen_at
+                    WHERE posthog_eventdefinition.last_seen_at < EXCLUDED.last_seen_at"#,
+        )
+        .bind(&batch.ids)
+        .bind(&batch.names)
+        .bind(&batch.team_ids)
+        .bind(&batch.project_ids)
+        .bind(&batch.last_seen_ats)
+        .execute(pool)
+        .await;
+
+        match result {
path: rust/property-defs-rs/src/lib.rs
user: login: eli-r-ph
body: Yeah bit of repetition there among the 3 batch types. I played with this and due to the use of different metrics keys everywhere per retry clause, it felt a bit awkward.

I'll land the PR and circle back on this change to DRY it up 👍 
created_at: 2025-03-28T17:38:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -120,6 +122,66 @@ pub struct PropertyDefinition {
     pub query_usage_30_day: Option<i64>,      // Deprecated
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
path: rust/property-defs-rs/src/types.rs
user: login: eli-r-ph
body: Done - all biz logic and types are extracted into their own module now for v2 batching 👍 
created_at: 2025-03-28T17:38:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -120,6 +122,66 @@ pub struct PropertyDefinition {
     pub query_usage_30_day: Option<i64>,      // Deprecated
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct PropertyDefinitionsBatch {
+    pub ids: Vec<Uuid>,
+    pub team_ids: Vec<i32>,
+    pub project_ids: Vec<i64>,
+    pub names: Vec<String>,
+    pub are_numerical: Vec<bool>,
+    pub event_types: Vec<i16>,
+    pub property_types: Vec<Option<i16>>,
+    pub group_type_indices: Vec<Option<i16>>,
+    // note: I left off deprecated fields we null out on writes
+}
+
+impl Default for PropertyDefinitionsBatch {
+    fn default() -> Self {
+        Self {
+            ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            team_ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            project_ids: Vec::with_capacity(WRITE_BATCH_SIZE),
+            names: Vec::with_capacity(WRITE_BATCH_SIZE),
+            are_numerical: Vec::with_capacity(WRITE_BATCH_SIZE),
+            property_types: Vec::with_capacity(WRITE_BATCH_SIZE),
+            event_types: Vec::with_capacity(WRITE_BATCH_SIZE),
+            group_type_indices: Vec::with_capacity(WRITE_BATCH_SIZE),
+        }
+    }
+}
+
+impl PropertyDefinitionsBatch {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn append(&mut self, pd: PropertyDefinition) -> bool {
+        let group_type_index: Option<i16> = match &pd.group_type_index {
+            Some(gt) => match gt {
+                GroupType::Resolved(_, gti) => Some(*gti as i16),
+                GroupType::Unresolved(_) => Some(-1_i16),
+            },
+            _ => Some(-1_i16),
+        };
+        let property_type: Option<i16> = pd.property_type.map(|pt| pt as i16);
+
+        self.ids.push(Uuid::now_v7());
+        self.team_ids.push(pd.team_id);
+        self.project_ids.push(pd.project_id);
+        self.names.push(pd.name);
+        self.are_numerical.push(pd.is_numerical);
+        self.property_types.push(property_type);
+        self.event_types.push(pd.event_type as i16);
+        self.group_type_indices.push(group_type_index);
+
+        self.ids.len() >= WRITE_BATCH_SIZE
path: rust/property-defs-rs/src/types.rs
user: login: eli-r-ph
body: new functions and clauses in the batch writes for `process_batch_v2` are in place and will land w/this PR ✅ 
created_at: 2025-03-28T17:39:10Z
author_association: CONTRIBUTOR

PR comments:
user: login: eli-r-ph
created_at: 2025-03-27T20:40:03Z
body: 👋 I'm marking this as ready for review. The only problem is a lint on `cargo sqlx prepare` which has been causing problems in local dev too. New test cases all pass locally. cc @pl 

I'll try to circle back with other Rust folks today to figure why I was able to get `cargo sqlx prepare` to run successfully in my local env but not in CI. The new macro-based queries are all in the test suite, and when the SQLX cache update runs successfully locally, it doesn't update any `.sqlx` query files which also struck me as odd.

☝️ cc @oliverb123 if you have context on all this. FYI I'm using Flox locally, and `sqlx-cli@0.8.3` (I have [a PR up](https://github.com/PostHog/posthog/pull/30479/) to ship that for the workspace too) but it hasn't been a blocker yet to date 🤞 
author_association: CONTRIBUTOR
user: login: eli-r-ph
created_at: 2025-03-28T00:52:07Z
body: 🎉 OK got this sorted out for the short term. Another PR on the way to clean up and script the `posthog/rust` migrations, dev/test bootstrapping, and to consolidate the `.sqlx` query cache at the top level for the whole workspace.
author_association: CONTRIBUTOR