Repository: frappe/frappe

Issue:
Fixture Import Order
## Description of the issue
When importing fixtures (e.g. on app install) the fixtures are neither imported in the order as given in hooks.py nor in alphabetical order ( due to #16773 ). The order actually seems to be random-per-environment, as it is based on pythons os.listdir
When one DocType links an other, the order becomes relevant. 
This will resolve https://discuss.frappe.io/t/how-to-specify-the-order-in-which-fixtures-are-to-be-imported/91409



## Steps to reproduce the issue

1. Export multiple Documents from different DocTypes with links between them
2. Try installing the App: it will fail in some

### Observed result
The fixtures are imported in a (pseudo) random order

### Expected result
- The fixtures are imported in a well defined order (e.g. alphanumerical)
- There is some mechanism in place (in hooks.py) to control the import order
e.g.:

- Add "filename_prefix" modifier, controls the file order manually 
- Add "order" modifier. The export will dynamically add a prefix to every file with the "order" modifier. 




Discussion:

Resolving patch:
diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@@ -241,9 +241,11 @@ def import_file(doctype, file_path, import_type, submit_after_import=False, cons
 	i.import_data()
 
 
-def import_doc(path, pre_process=None):
+def import_doc(path, pre_process=None, sort=False):
 	if os.path.isdir(path):
 		files = [os.path.join(path, f) for f in os.listdir(path)]
+		if sort:
+			files.sort()
 	else:
 		files = [path]
 
diff --git a/frappe/utils/fixtures.py b/frappe/utils/fixtures.py
--- a/frappe/utils/fixtures.py
+++ b/frappe/utils/fixtures.py
@@ -38,7 +38,7 @@ def import_fixtures(app):
 
 		file_path = frappe.get_app_path(app, "fixtures", fname)
 		try:
-			import_doc(file_path)
+			import_doc(file_path, sort=True)
 		except (ImportError, frappe.DoesNotExistError) as e:
 			# fixture syncing for missing doctypes
 			print(f"Skipping fixture syncing from the file {fname}. Reason: {e}")
@@ -67,20 +67,32 @@ def export_fixtures(app=None):
 	else:
 		apps = frappe.get_installed_apps()
 	for app in apps:
-		for fixture in frappe.get_hooks("fixtures", app_name=app):
+		fixture_auto_order = bool(next(iter(frappe.get_hooks("fixture_auto_order", app_name=app)), False))
+		fixtures = frappe.get_hooks("fixtures", app_name=app)
+		for index, fixture in enumerate(fixtures, start=1):
 			filters = None
 			or_filters = None
 			if isinstance(fixture, dict):
 				filters = fixture.get("filters")
 				or_filters = fixture.get("or_filters")
+				prefix = fixture.get("prefix")
 				fixture = fixture.get("doctype") or fixture.get("dt")
 			print(f"Exporting {fixture} app {app} filters {(filters if filters else or_filters)}")
 			if not os.path.exists(frappe.get_app_path(app, "fixtures")):
 				os.mkdir(frappe.get_app_path(app, "fixtures"))
 
+			filename = frappe.scrub(fixture)
+			if prefix:
+				filename = f"{prefix}_{filename}"
+			if fixture_auto_order:
+				number_of_digits = len(str(len(fixtures)))
+				# add zero padding so files can be sorted lexicographically with filename.
+				file_number = str(index).zfill(number_of_digits)
+				filename = f"{file_number}_{filename}"
+
 			export_json(
 				fixture,
-				frappe.get_app_path(app, "fixtures", frappe.scrub(fixture) + ".json"),
+				frappe.get_app_path(app, "fixtures", filename + ".json"),
 				filters=filters,
 				or_filters=or_filters,
 				order_by="idx asc, creation asc",


Test patch:
diff --git a/frappe/tests/test_hooks.py b/frappe/tests/test_hooks.py
--- a/frappe/tests/test_hooks.py
+++ b/frappe/tests/test_hooks.py
@@ -95,6 +95,70 @@ def test_ignore_links_on_delete(self):
 
 		event.delete()
 
+	def test_fixture_prefix(self):
+		import os
+		import shutil
+		from frappe import hooks
+		from frappe.utils.fixtures import export_fixtures
+
+		app = "frappe"
+		if(os.path.isdir(frappe.get_app_path(app, "fixtures"))):
+			shutil.rmtree(frappe.get_app_path(app, "fixtures"))
+
+		# use any set of core doctypes for test purposes
+		hooks.fixtures = [
+			{"dt": "User"},
+			{"dt": "Contact"},
+			{"dt": "Role"},
+		]
+		hooks.fixture_auto_order = False
+		# every call to frappe.get_hooks loads the hooks module into cache
+		# therefor the cache has to be invalidated after every manual overwriting of hooks
+		# TODO replace with a more elegant solution if there is one or build a util function for this purpose
+		if frappe._load_app_hooks.__wrapped__ in frappe.local.request_cache.keys():
+			del frappe.local.request_cache[frappe._load_app_hooks.__wrapped__]
+		self.assertEqual([False], frappe.get_hooks("fixture_auto_order", app_name=app))
+		self.assertEqual([
+			{"dt": "User"},
+			{"dt": "Contact"},
+			{"dt": "Role"},
+		], frappe.get_hooks("fixtures", app_name=app))
+
+		export_fixtures(app)
+		# use assertCountEqual (replaced assertItemsEqual), beacuse os.listdir might return the list in a different order, depending on OS
+		self.assertCountEqual(["user.json", "contact.json", "role.json"], os.listdir(frappe.get_app_path(app, "fixtures")))
+
+
+		hooks.fixture_auto_order = True
+		del frappe.local.request_cache[frappe._load_app_hooks.__wrapped__]
+		self.assertEqual([True], frappe.get_hooks("fixture_auto_order", app_name=app))
+
+		shutil.rmtree(frappe.get_app_path(app, "fixtures"))
+		export_fixtures(app)
+		self.assertCountEqual(["1_user.json", "2_contact.json", "3_role.json"], os.listdir(frappe.get_app_path(app, "fixtures")))
+
+
+		hooks.fixtures = [
+			{
+				"dt": "User",
+				"prefix": "my_prefix"},
+			{"dt": "Contact"},
+			{"dt": "Role"},
+		]
+		hooks.fixture_auto_order = False
+
+		del frappe.local.request_cache[frappe._load_app_hooks.__wrapped__]
+		shutil.rmtree(frappe.get_app_path(app, "fixtures"))
+		export_fixtures(app)
+		self.assertCountEqual(["my_prefix_user.json", "contact.json", "role.json"], os.listdir(frappe.get_app_path(app, "fixtures")))
+
+
+		hooks.fixture_auto_order = True
+		del frappe.local.request_cache[frappe._load_app_hooks.__wrapped__]
+		shutil.rmtree(frappe.get_app_path(app, "fixtures"))
+		export_fixtures(app)
+		self.assertCountEqual(["1_my_prefix_user.json", "2_contact.json", "3_role.json"], os.listdir(frappe.get_app_path(app, "fixtures")))
+
 
 class TestAPIHooks(FrappeAPITestCase):
 	def test_auth_hook(self):