Repository: metabase/metabase
Pull request #56671: Don't update the "active" index table from the "reindex" job
Author: login: nvoxland
Labels: name: backport
description: Automatically create PR on current release branch on merge
name: .Team/AdminWebapp
description: DEPRECATED  Please use .Team/UXWest instead
name: double-backport
Created: 2025-04-14T14:34:16Z | Merged: 2025-04-21T14:26:53Z

### Description

When there are realtime updates to items, we update both the "active" index and the "pending" index still being created (if one exists). We need both as the re-indexing might still take some time, but it may also have already progressed past the corresponding entities.

### Checklist

- [X] Tests have been added/updated to cover changes in this PR



Diff:
diff --git a/src/metabase/search/api.clj b/src/metabase/search/api.clj
index 4d534fed9e91..cfa35bb4deed 100644
--- a/src/metabase/search/api.clj
+++ b/src/metabase/search/api.clj
@@ -12,6 +12,7 @@
    [metabase.request.core :as request]
    [metabase.search.config :as search.config]
    [metabase.search.core :as search]
+   [metabase.search.ingestion :as ingestion]
    [metabase.search.task.search-index :as task.search-index]
    [metabase.task :as task]
    [metabase.util :as u]
@@ -61,9 +62,9 @@
   "This will trigger an immediate reindexing, if we are using search index."
   []
   (api/check-superuser)
-  (if  (search/supports-index?)
+  (if (search/supports-index?)
     ;; The job appears to wait on the main thread when run from tests, so, unfortunately, testing this branch is hard.
-    (if (and (task/job-exists? task.search-index/reindex-job-key) (not config/is-test?))
+    (if (and (task/job-exists? task.search-index/reindex-job-key) (or (not ingestion/*force-sync*) (config/is-test?)))
       (do (task/trigger-now! task.search-index/reindex-job-key) {:message "task triggered"})
       (do (task.search-index/reindex!) {:message "done"}))
 
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 5c1134fcb14b..9b66624c62ea 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -143,11 +143,14 @@
          t2/query
          (into #{} (map :model)))))
 
+(defn- populate-index! [context]
+  (search.index/index-docs! context (search.ingestion/searchable-documents)))
+
 (defmethod search.engine/init! :search.engine/appdb
   [_ {:keys [re-populate?] :as opts}]
   (let [created? (search.index/ensure-ready! opts)]
     (when (or created? re-populate?)
-      (search.ingestion/populate-index! :search.engine/appdb))))
+      (populate-index! :search/updating))))
 
 (defmethod search.engine/reindex! :search.engine/appdb
   [_ {:keys [in-place?]}]
@@ -157,5 +160,5 @@
       ;; keep the current table, just delete its contents
       (t2/delete! table))
     (search.index/maybe-create-pending!))
-  (u/prog1 (search.ingestion/populate-index! :search.engine/appdb)
+  (u/prog1 (populate-index! (if in-place? :search/updating :search/reindexing))
     (search.index/activate-table!)))
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 71657ec5f046..e19041fbee42 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -243,7 +243,7 @@
 
 (defn- batch-update!
   "Create the given search index entries in bulk"
-  [documents]
+  [context documents]
   ;; Protect against tests that nuke the appdb
   (when config/is-test?
     (when-let [table (active-table)]
@@ -256,21 +256,25 @@
         (swap! *indexes* assoc :pending nil))))
 
   (let [entries          (map document->entry documents)
-        ;; Optimization idea: if the updates are coming from the re-indexing worker, skip updating the active table.
-        ;;                    this should give a close to 2x speed-up as insertion is the bottleneck, and most of the
-        ;;                    updates will be no-ops in any case.
-        active-updated?  (safe-batch-upsert! (active-table) entries)
+        ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
+        active-updated?  (when-not (= context :search/reindexing) (safe-batch-upsert! (active-table) entries))
         pending-updated? (safe-batch-upsert! (pending-table) entries)]
     (when (or active-updated? pending-updated?)
       (u/prog1 (->> entries (map :model) frequencies)
-        (log/trace "indexed documents" <>)))))
+        (log/trace "indexed documents for " <>)))))
 
-(defmethod search.engine/consume! :search.engine/appdb [_engine document-reducible]
+(defn index-docs!
+  "Indexes the documents. The context should be :search/updating or :search/reindexing.
+   Context should be :search/updating or :search/reindexing to help control how to manage the updates"
+  [context document-reducible]
   (transduce (comp (partition-all insert-batch-size)
-                   (map batch-update!))
+                   (map (partial batch-update! context)))
              (partial merge-with +)
              document-reducible))
 
+(defmethod search.engine/update! :search.engine/appdb [_engine document-reducible]
+  (index-docs! :search/updating document-reducible))
+
 (defmethod search.engine/delete! :search.engine/appdb [_engine search-model ids]
   (doseq [table-name [(active-table) (pending-table)] :when table-name]
     (t2/delete! table-name :model search-model :model_id [:in ids])))
diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.clj
index c216b7f2ce20..92fd98345ba5 100644
--- a/src/metabase/search/engine.clj
+++ b/src/metabase/search/engine.clj
@@ -29,8 +29,8 @@
   {:result (dissoc result :score)
    :score  (:score result)})
 
-(defmulti consume!
-  "Updates the search index by consuming the documents from the given reducible."
+(defmulti update!
+  "Updates the existing search index by consuming the documents from the given reducible."
   {:arglists '([search-engine document-reducible])}
   (fn [search-engine _document-reducible]
     search-engine))
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index a9bab6df293c..cd51b689dafd 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -101,10 +101,10 @@
          (m/distinct-by (juxt :id :model))
          (map ->document)))))
 
-(defn populate-index!
-  "Go over all searchable items and populate the index with them."
-  [engine]
-  (search.engine/consume! engine (query->documents (search-items-reducible))))
+(defn searchable-documents
+  "Return all existing searchable documents from the database."
+  []
+  (query->documents (search-items-reducible)))
 
 (def ^:dynamic *force-sync*
   "Force ingestion to happen immediately, on the same thread."
@@ -114,16 +114,16 @@
   "Used by tests to disable updates, for example when testing migrations, where the schema is wrong."
   false)
 
-(defn consume!
-  "Update all active engines' indexes with the given documents"
+(defn update!
+  "Update all active engines' existing indexes with the given documents"
   [documents-reducible]
   (when-let [engines (seq (search.engine/active-engines))]
     (if (= 1 (count engines))
-      (search.engine/consume! (first engines) documents-reducible)
+      (search.engine/update! (first engines) documents-reducible)
       ;; TODO um, multiplexing over the reducible awkwardly feels strange. We at least use a magic number for now.
       (doseq [batch (eduction (partition-all 150) documents-reducible)
               e     engines]
-        (search.engine/consume! e batch)))))
+        (search.engine/update! e batch)))))
 
 (defn bulk-ingest!
   "Process the given search model updates. Returns the number of search index entries that get updated as a result."
@@ -133,7 +133,7 @@
        ;; init collection is only for clj-kondo, as we know that the list is non-empty
        (reduce u/rconcat [])
        query->documents
-       consume!))
+       update!))
 
 (defn- track-queue-size! []
   (analytics/set! :metabase-search/queue-size (.size queue)))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index 069061571f84..b1b191aa6f60 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -53,15 +53,14 @@
                         :model/Card       {}            {:name "Projected Satisfaction"         :collection_id col-id#}
                         :model/Database   {db-id# :id}  {:name "Indexed Database"}
                         :model/Table      {}            {:name "Indexed Table", :db_id db-id#}]
-           (search.index/reset-index!)
-           (search.ingestion/populate-index! :search.engine/appdb)
+           (search.engine/reindex! :search.engine/appdb {:in-place? true})
            ~@body)))))
 
 (deftest idempotent-test
   (with-index
     (let [count-rows  (fn [] (t2/count (search.index/active-table)))
           rows-before (count-rows)]
-      (search.ingestion/populate-index! :search.engine/appdb)
+      (search.engine/reindex! :search.engine/appdb {:in-place? true})
       (is (= rows-before (count-rows))))))
 
 (deftest incremental-update-test
@@ -153,7 +152,7 @@
 
 (defn ingest!
   [model where-clause]
-  (#'search.engine/consume!
+  (#'search.engine/update!
    :search.engine/appdb
    (#'search.ingestion/query->documents
     (#'search.ingestion/spec-index-reducible model where-clause))))
diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.clj
index ba46e40ea512..81a01021bc5b 100644
--- a/test/metabase/search/appdb/scoring_test.clj
+++ b/test/metabase/search/appdb/scoring_test.clj
@@ -196,7 +196,7 @@
                 _               (t2/insert! :model/Card (concat (repeatedly 20 #(card-with-view 0))
                                                                 (for [i (range 1 81)]
                                                                   (card-with-view i))))
-                _               (search.ingestion/consume!
+                _               (search.ingestion/update!
                                  (#'search.ingestion/query->documents
                                   (#'search.ingestion/spec-index-reducible "card" [:= :this.name search-term])))
                 first-result-id (-> (search-results* search-term) first second)]
diff --git a/test/metabase/search/test_util.clj b/test/metabase/search/test_util.clj
index 8d34720c643a..7ef73e3c4b5b 100644
--- a/test/metabase/search/test_util.clj
+++ b/test/metabase/search/test_util.clj
@@ -13,6 +13,13 @@
 
 (def ^:dynamic *user-ctx* nil)
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
+(defmacro with-sync-search-indexing
+  "Perform all search indexing synchronously."
+  [& body]
+  `(binding [metabase.search.ingestion/*force-sync* true]
+     ~@body))
+
 #_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defmacro with-temp-index-table
   "Create a temporary index table for the duration of the body."
@@ -20,7 +27,7 @@
   `(when (search/supports-index?)
      (search.index/with-temp-index-table
       ;; We need ingestion to happen on the same thread so that it uses the right search index.
-       (binding [metabase.search.ingestion/*force-sync* true]
+       (with-sync-search-indexing
          ~@body))))
 
 #_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}


Reviews:
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T10:10:44Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T10:17:11Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T10:19:36Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T21:55:44Z
user: login: nvoxland
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T05:30:04Z
user: login: nvoxland
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T05:33:21Z
user: login: nvoxland
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T05:34:42Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:02:57Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:03:33Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:06:49Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:07:28Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:12:28Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:14:25Z
user: login: crisptrutski
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:15:36Z
user: login: crisptrutski
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T08:15:41Z
user: login: nvoxland
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T14:05:39Z
user: login: edpaget
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T13:42:04Z

Review comments:
diff_hunk: @@ -639,7 +639,7 @@
         ;; Ideally this would be DRY with the actual specification some way, but since this is a stop-gap solution, we
         ;; decided not to complicate the solution further to accomplish this.
         (search/bulk-ingest! (for [search-model ["card" "dataset" "metric"]]
-                               [search-model [:= :this.id id]])))
+                               [search-model [:= :this.id id]]) false))
path: src/metabase/api/card.clj
user: login: crisptrutski
body: Having a naked boolean at the end feels a bit opaque to me, how about making it a kwarg map?

One downside would be that you're not forced to set the value, but I think it's OK having a default.
created_at: 2025-04-15T10:10:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -115,24 +115,24 @@
 
 (defn consume!
   "Update all active engines' indexes with the given documents"
-  [documents-reducible]
+  [documents-reducible re-indexing?]
   (when-let [engines (seq (search.engine/active-engines))]
     (if (= 1 (count engines))
-      (search.engine/consume! (first engines) documents-reducible)
+      (search.engine/consume! (first engines) documents-reducible re-indexing?)
       ;; TODO um, multiplexing over the reducible awkwardly feels strange. We at least use a magic number for now.
       (doseq [batch (eduction (partition-all 150) documents-reducible)
               e     engines]
-        (search.engine/consume! e batch)))))
+        (search.engine/consume! e batch re-indexing?)))))
 
 (defn bulk-ingest!
   "Process the given search model updates. Returns the number of search index entries that get updated as a result."
-  [updates]
+  [updates re-indexing?]
   (->> (for [[search-model where-clauses] (u/group-by first second updates)]
          (spec-index-reducible search-model (into [:or] (distinct where-clauses))))
        ;; init collection is only for clj-kondo, as we know that the list is non-empty
        (reduce u/rconcat [])
        query->documents
-       consume!))
+       (#(consume! % re-indexing?))))
path: src/metabase/search/ingestion.clj
user: login: crisptrutski
body: Oh man I hate having to do this kind of double nesting.

Thinking whether it could make sense to swap the args around because the implementation of consume also find the order awkward. Feels like a smell to not have the list argument last if its used in "pipelines" like this.

Rather than `re-indexing?` flag you could make have a `context` keyword argument, with values like `:indexing`, `:re-indexing`, and `:update`? Taking that first feels natural to me.
created_at: 2025-04-15T10:17:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -639,7 +639,7 @@
         ;; Ideally this would be DRY with the actual specification some way, but since this is a stop-gap solution, we
         ;; decided not to complicate the solution further to accomplish this.
         (search/bulk-ingest! (for [search-model ["card" "dataset" "metric"]]
-                               [search-model [:= :this.id id]])))
+                               [search-model [:= :this.id id]]) false))
path: src/metabase/api/card.clj
user: login: crisptrutski
body: I actually prefer the second suggestion of having `context` argument first, it feels more natural and ergonomic, and makes the argument self describing without needing to be a keyword arg.
created_at: 2025-04-15T10:19:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -150,9 +150,9 @@
       (search.ingestion/populate-index! :search.engine/appdb))))
 
 (defmethod search.engine/reindex! :search.engine/appdb
-  [_ {:keys [in-place?]}]
+  [_ _]
   (search.index/ensure-ready!)
-  (if in-place?
+  (if search.index/*temp-index-tables*
path: src/metabase/search/appdb/core.clj
user: login: crisptrutski
body: Why make this change? Would prefer to only perform this hack when explicitly asked for.
created_at: 2025-04-16T21:52:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -103,7 +104,7 @@
 (defn populate-index!
   "Go over all searchable items and populate the index with them."
   [engine]
-  (search.engine/consume! engine (query->documents (search-items-reducible))))
+  (search.engine/consume! engine (query->documents (search-items-reducible)) (not search.index/*temp-index-tables*)))
path: src/metabase/search/ingestion.clj
user: login: crisptrutski
body: Not ideal to "lie" with this argument in the temp case, why is this necessary? 
created_at: 2025-04-16T21:53:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,18 +258,16 @@
         (swap! *indexes* assoc :pending nil))))
 
   (let [entries          (map document->entry documents)
-        ;; Optimization idea: if the updates are coming from the re-indexing worker, skip updating the active table.
-        ;;                    this should give a close to 2x speed-up as insertion is the bottleneck, and most of the
-        ;;                    updates will be no-ops in any case.
-        active-updated?  (safe-batch-upsert! (active-table) entries)
+        ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
path: src/metabase/search/appdb/index.clj
user: login: crisptrutski
body: Worth mentioning that these inserts are the bottleneck, so we're doing this for a large expected speed-up.
created_at: 2025-04-16T21:54:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -40,7 +40,9 @@
 
 (defonce ^:dynamic ^:private *indexes* (atom {:active nil, :pending nil}))
 
-(def ^:private ^:dynamic *mocking-tables* false)
+(def ^:dynamic *temp-index-tables*
path: src/metabase/search/appdb/index.clj
user: login: crisptrutski
body: Not great leaking this dynamic var outside the namespace, and both places this is don't seem like good ideas to me, at first glance.
created_at: 2025-04-16T21:55:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -639,7 +639,7 @@
         ;; Ideally this would be DRY with the actual specification some way, but since this is a stop-gap solution, we
         ;; decided not to complicate the solution further to accomplish this.
         (search/bulk-ingest! (for [search-model ["card" "dataset" "metric"]]
-                               [search-model [:= :this.id id]])))
+                               [search-model [:= :this.id id]]) false))
path: src/metabase/api/card.clj
user: login: nvoxland
body: I ended up with a combination of adding the context arg as a 1st argument, but also adjusting the search API a bit to limit the need for passing it and better isolating it to the appdb implementation of that API.

LMK what you think of the new version
created_at: 2025-04-18T05:30:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -150,9 +150,9 @@
       (search.ingestion/populate-index! :search.engine/appdb))))
 
 (defmethod search.engine/reindex! :search.engine/appdb
-  [_ {:keys [in-place?]}]
+  [_ _]
   (search.index/ensure-ready!)
-  (if in-place?
+  (if search.index/*temp-index-tables*
path: src/metabase/search/appdb/core.clj
user: login: nvoxland
body: I had made the change because the earlier version needed was seeing if combining the in-place? and "mock-table" setup for tests would help fix the failing tests. The switch to making it public was also part of that since it ended up needing both core and indexing that needed access to it. 

But in the end, I was able to put it back
created_at: 2025-04-18T05:33:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,18 +258,16 @@
         (swap! *indexes* assoc :pending nil))))
 
   (let [entries          (map document->entry documents)
-        ;; Optimization idea: if the updates are coming from the re-indexing worker, skip updating the active table.
-        ;;                    this should give a close to 2x speed-up as insertion is the bottleneck, and most of the
-        ;;                    updates will be no-ops in any case.
-        active-updated?  (safe-batch-upsert! (active-table) entries)
+        ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
path: src/metabase/search/appdb/index.clj
user: login: nvoxland
body: Yeah, getting to this part of the change is overall point of this PR and I prioritized it for the hopeful performance boost.
created_at: 2025-04-18T05:34:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,18 +258,16 @@
         (swap! *indexes* assoc :pending nil))))
 
   (let [entries          (map document->entry documents)
-        ;; Optimization idea: if the updates are coming from the re-indexing worker, skip updating the active table.
-        ;;                    this should give a close to 2x speed-up as insertion is the bottleneck, and most of the
-        ;;                    updates will be no-ops in any case.
-        active-updated?  (safe-batch-upsert! (active-table) entries)
+        ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
path: src/metabase/search/appdb/index.clj
user: login: crisptrutski
body: Sorry I just meant worth mentioning in this comment
created_at: 2025-04-18T08:02:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,21 +256,25 @@
         (swap! *indexes* assoc :pending nil))))
 
   (let [entries          (map document->entry documents)
-        ;; Optimization idea: if the updates are coming from the re-indexing worker, skip updating the active table.
-        ;;                    this should give a close to 2x speed-up as insertion is the bottleneck, and most of the
-        ;;                    updates will be no-ops in any case.
-        active-updated?  (safe-batch-upsert! (active-table) entries)
+        ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
+        active-updated?  (when-not (= context :search/reindexing) (safe-batch-upsert! (active-table) entries))
         pending-updated? (safe-batch-upsert! (pending-table) entries)]
     (when (or active-updated? pending-updated?)
       (u/prog1 (->> entries (map :model) frequencies)
-        (log/trace "indexed documents" <>)))))
+        (log/trace "indexed documents for " <>)))))
 
-(defmethod search.engine/consume! :search.engine/appdb [_engine document-reducible]
+(defn index-docs!
+  "Indexes the documents. The context should be :search/updating or :search/reindexing.
+   Context should be :search/updating or :search/reindexing to help control how to manage the updates"
+  [context document-reducible]
   (transduce (comp (partition-all insert-batch-size)
-                   (map batch-update!))
+                   (map #(batch-update! context %)))
path: src/metabase/search/appdb/index.clj
user: login: crisptrutski
body: nit: a bit more natural with `partial` IMHO
created_at: 2025-04-18T08:03:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -53,15 +53,14 @@
                         :model/Card       {}            {:name "Projected Satisfaction"         :collection_id col-id#}
                         :model/Database   {db-id# :id}  {:name "Indexed Database"}
                         :model/Table      {}            {:name "Indexed Table", :db_id db-id#}]
-           (search.index/reset-index!)
-           (search.ingestion/populate-index! :search.engine/appdb)
+           (search.engine/reindex! :search.engine/appdb {:in-place? true})
path: test/metabase/search/appdb/index_test.clj
user: login: crisptrutski
body: Creating a new index feels cleaner to me, like starting up the app again. Curious as to your reasoning, but don't think the distinction is too important in practice.
created_at: 2025-04-18T08:06:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -53,15 +53,14 @@
                         :model/Card       {}            {:name "Projected Satisfaction"         :collection_id col-id#}
                         :model/Database   {db-id# :id}  {:name "Indexed Database"}
                         :model/Table      {}            {:name "Indexed Table", :db_id db-id#}]
-           (search.index/reset-index!)
-           (search.ingestion/populate-index! :search.engine/appdb)
+           (search.engine/reindex! :search.engine/appdb {:in-place? true})
            ~@body)))))
 
 (deftest idempotent-test
   (with-index
     (let [count-rows  (fn [] (t2/count (search.index/active-table)))
           rows-before (count-rows)]
-      (search.ingestion/populate-index! :search.engine/appdb)
+      (search.engine/reindex! :search.engine/appdb {:in-place? true})
path: test/metabase/search/appdb/index_test.clj
user: login: crisptrutski
body: For this test I prefer the increased coverage of fully re-bootstrapping.
created_at: 2025-04-18T08:07:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -63,7 +63,7 @@
   (api/check-superuser)
   (if  (search/supports-index?)
     ;; The job appears to wait on the main thread when run from tests, so, unfortunately, testing this branch is hard.
-    (if (and (task/job-exists? task.search-index/reindex-job-key) (not config/is-test?))
+    (if (and (task/job-exists? task.search-index/reindex-job-key) (not ingestion/*force-sync*))
path: src/metabase/search/api.clj
user: login: crisptrutski
body: We need this condition to hold whenever we're using the test running, even if that particular test has not forced sync for *all* mutations.

Perhaps this works in practice for now, but it's pretty brittle. I'm not sure of a general way to test whether you're inside a test besides checking the current thread name, but adding that with an `or` is better than nothing I guess.


created_at: 2025-04-18T08:12:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -63,7 +63,7 @@
   (api/check-superuser)
   (if  (search/supports-index?)
     ;; The job appears to wait on the main thread when run from tests, so, unfortunately, testing this branch is hard.
-    (if (and (task/job-exists? task.search-index/reindex-job-key) (not config/is-test?))
+    (if (and (task/job-exists? task.search-index/reindex-job-key) (not ingestion/*force-sync*))
path: src/metabase/search/api.clj
user: login: crisptrutski
body: Or just `(or config/is-test? ingestion/*force-sync*)` and we don't need to worry about the thread naming convention running, and instead risk tests that fail in the REPL in the future.

Not going to block on this as I don't anticipate other tests calling this API in the future, it's there for debugging really.
created_at: 2025-04-18T08:14:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -100,10 +100,10 @@
          (m/distinct-by (juxt :id :model))
          (map ->document)))))
 
-(defn populate-index!
-  "Go over all searchable items and populate the index with them."
-  [engine]
-  (search.engine/consume! engine (query->documents (search-items-reducible))))
+(defn searchable-documents
path: src/metabase/search/ingestion.clj
user: login: crisptrutski
body: Nice work on refining the structure and function naming
created_at: 2025-04-18T08:15:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -256,18 +258,16 @@
         (swap! *indexes* assoc :pending nil))))
 
   (let [entries          (map document->entry documents)
-        ;; Optimization idea: if the updates are coming from the re-indexing worker, skip updating the active table.
-        ;;                    this should give a close to 2x speed-up as insertion is the bottleneck, and most of the
-        ;;                    updates will be no-ops in any case.
-        active-updated?  (safe-batch-upsert! (active-table) entries)
+        ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
path: src/metabase/search/appdb/index.clj
user: login: nvoxland
body: 💯 Just agreeing with you 
created_at: 2025-04-18T14:05:39Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-04-14T15:32:33Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://raster.shields.io/badge/108330_tests-6fc038d6-purple)](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment) &ensp; [![Static Badge](https://raster.shields.io/badge/0-failed-crimson)](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/4-flaky-red)](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://raster.shields.io/badge/0-quarantined-yellow)](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=QUARANTINED)


| Flaky Test | Failure Summary | Logs |
| --- | --- | --- |
| <code>update-timeline-event-test</code> | The expected timestamp was not equal to the actual timestamp when updating a timeline event. | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=FAILURE&runId=43181a0d-4ae3-5e15-a98a-525188aa6417) |
| <code>identity-hash-test</code> | Card hashes are not matching as expected, expected "5199edf0" but got "b36d7acf" and "01c7ccaf". | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=FAILURE&runId=a419f73a-43d6-5de7-bbf9-f01301a73997) |
| <code>identity-hash-test</code> | The calculated hash of a dashboard does not match the expected hash. | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=FAILURE&runId=cb18946c-7d3c-53f1-b40e-daedbc492e41) |
| <code>scenarios > admin > datamodel > editor field settings should allow changing the field foreign key target</code> | The test timed out while trying to find an element with the placeholder text 'Select a target'. | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment&runConclusion=FAILURE&runId=e66a7164-5e1d-55ea-8399-cfc6eb2b7bf6) |
<sub>

[View Full Report ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56671?repo=metabase/metabase&commitHash=6fc038d61e61833ccb90f9ee693a53251b777c23&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE
user: login: crisptrutski
created_at: 2025-04-15T10:51:55Z
body: Not going to block on stylistic preferences, but holding back the tick since there are so many failing tests to look into
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-18T15:16:47Z
body: 
  ## e2e tests failed on `6fc038d61e61833ccb90f9ee693a53251b777c23-1`

  [e2e test run](https://github.com/metabase/metabase/actions/runs/14537090908?pr=56671)

  File | Test Name
  ---- | ---------
  `editor.cy.spec.js` | scenarios > admin > datamodel > editor > field settings > should allow changing the field foreign key target

author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-21T15:12:37Z
body: 🚀 This should also be released by [v0.54.4](https://github.com/metabase/metabase/milestone/329)
author_association: CONTRIBUTOR