Repository: frappe/frappe

Issue:
sync_global_search should do bulk insert
I'm in the process of importing the data from our current system into ERPNext, which includes 60,000+ items. I am having to do these 5000 lines at a time because of the timeout limit, but I have also noticed that `frappe.utils.global_search.sync_global_search` seems to be called for every single new record. This is a huge drain on resources and as there are only two workers on the background jobs I have to wait for these to be processed before doing anything else.

I think the system should be smart enough to know that if an import is running, then sync_global_search should not run until the import has finished - and then it should only run once. 


Discussion:
While this is improved and the original problem you have mentioned is gone; it still does one insert at a time 😄 Maybe this can be further optimized to do bulk insert!

https://github.com/frappe/frappe/blob/3a34bfc52035b2ff23091d827d8f41a8c7f49f30/frappe/utils/global_search.py#L355-L357 

Resolving patch:
diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py
--- a/frappe/utils/global_search.py
+++ b/frappe/utils/global_search.py
@@ -11,6 +11,7 @@
 from frappe.model.base_document import get_controller
 from frappe.utils import cint, strip_html_tags
 from frappe.utils.data import cstr
+from frappe.utils.deprecations import deprecated
 from frappe.utils.html_utils import unescape_html
 
 HTML_TAGS_PATTERN = re.compile(r"(?s)<[\s]*(script|style).*?</\1>")
@@ -373,11 +374,53 @@ def sync_global_search():
 	:param flags:
 	:return:
 	"""
-	while frappe.cache.llen("global_search_queue") > 0:
-		# rpop to follow FIFO
-		# Last one should override all previous contents of same document
-		value = json.loads(frappe.cache.rpop("global_search_queue").decode("utf-8"))
-		sync_value(value)
+	from itertools import islice
+
+	def get_search_queue_item_generator():
+		while value := frappe.cache.rpop("global_search_queue"):
+			yield value
+
+	item_generator = get_search_queue_item_generator()
+	while search_items := tuple(islice(item_generator, 10_000)):
+		values = _get_deduped_search_item_values(search_items)
+		sync_values(values)
+
+
+def _get_deduped_search_item_values(items):
+	from collections import OrderedDict
+
+	values_dict = OrderedDict()
+	for item in items:
+		item_json = item.decode("utf-8")
+		item_dict = json.loads(item_json)
+		key = (item_dict["doctype"], item_dict["name"])
+		values_dict[key] = tuple(item_dict.values())
+
+	return values_dict.values()
+
+
+def sync_values(values: list):
+	from pypika.terms import Values
+
+	GlobalSearch = frappe.qb.Table("__global_search")
+	conflict_fields = ["content", "published", "title", "route"]
+
+	query = (
+		frappe.qb.into(GlobalSearch).columns(["doctype", "name"] + conflict_fields).insert(*values)
+	)
+
+	if frappe.db.db_type == "postgres":
+		query = query.on_conflict(GlobalSearch.doctype, GlobalSearch.name)
+
+	for field in conflict_fields:
+		if frappe.db.db_type == "mariadb":
+			query = query.on_duplicate_key_update(GlobalSearch[field], Values(field))
+		elif frappe.db.db_type == "postgres":
+			query = query.do_update(GlobalSearch[field])
+		else:
+			raise NotImplementedError
+
+	query.run()
 
 
 def sync_value_in_queue(value):
@@ -386,10 +429,11 @@ def sync_value_in_queue(value):
 		frappe.cache.lpush("global_search_queue", json.dumps(value))
 	except redis.exceptions.ConnectionError:
 		# not connected, sync directly
-		sync_value(value)
+		sync_values((value,))
 
 
-def sync_value(value):
+@deprecated
+def sync_value(value: dict):
 	"""
 	Sync a given document to global search
 	:param value: dict of { doctype, name, content, published, title, route }


Test patch: