Repository: conan-io/conan

Issue:
Apply migrations in the proper order
Currently, migrations are being applied in reverse order. They should be applied from the lowest version to the higher one, so each migration is applied on top of all the previous ones (probably related with this bug: https://github.com/conan-io/conan/issues/4866)


```python
        if old_version < Version("0.25"):
            ...

        if old_version < Version("1.0"):
            _migrate_lock_files(self.cache, self.out)

        if old_version < Version("1.12.0"):
            migrate_plugins_to_hooks(self.cache)

        if old_version < Version("1.13.0"):
            # MIGRATE LOCAL CACHE TO GENERATE MISSING METADATA.json
            _migrate_create_metadata(self.cache, self.out)

        if old_version < Version("1.14.0"):
            migrate_config_install(self.cache)

```



Discussion:

Resolving patch:
diff --git a/conans/client/migrations.py b/conans/client/migrations.py
--- a/conans/client/migrations.py
+++ b/conans/client/migrations.py
@@ -51,8 +51,25 @@ def _make_migrations(self, old_version):
         if old_version is None:
             return
 
-        if old_version < Version("1.14.0"):
-            migrate_config_install(self.cache)
+        if old_version < Version("0.25"):
+            from conans.paths import DEFAULT_PROFILE_NAME
+            default_profile_path = os.path.join(self.cache.conan_folder, PROFILES_FOLDER,
+                                                DEFAULT_PROFILE_NAME)
+            if not os.path.exists(default_profile_path):
+                self.out.warn("Migration: Moving default settings from %s file to %s"
+                              % (CONAN_CONF, DEFAULT_PROFILE_NAME))
+                conf_path = os.path.join(self.cache.conan_folder, CONAN_CONF)
+
+                migrate_to_default_profile(conf_path, default_profile_path)
+
+                self.out.warn("Migration: export_source cache new layout")
+                migrate_c_src_export_source(self.cache, self.out)
+
+        if old_version < Version("1.0"):
+            _migrate_lock_files(self.cache, self.out)
+
+        if old_version < Version("1.12.0"):
+            migrate_plugins_to_hooks(self.cache)
 
         if old_version < Version("1.13.0"):
             old_settings = """
@@ -127,25 +144,11 @@ def _make_migrations(self, old_version):
             # MIGRATE LOCAL CACHE TO GENERATE MISSING METADATA.json
             _migrate_create_metadata(self.cache, self.out)
 
-        if old_version < Version("1.12.0"):
-            migrate_plugins_to_hooks(self.cache)
-
-        if old_version < Version("1.0"):
-            _migrate_lock_files(self.cache, self.out)
-
-        if old_version < Version("0.25"):
-            from conans.paths import DEFAULT_PROFILE_NAME
-            default_profile_path = os.path.join(self.cache.conan_folder, PROFILES_FOLDER,
-                                                DEFAULT_PROFILE_NAME)
-            if not os.path.exists(default_profile_path):
-                self.out.warn("Migration: Moving default settings from %s file to %s"
-                              % (CONAN_CONF, DEFAULT_PROFILE_NAME))
-                conf_path = os.path.join(self.cache.conan_folder, CONAN_CONF)
-
-                migrate_to_default_profile(conf_path, default_profile_path)
+        if old_version < Version("1.14.0"):
+            migrate_config_install(self.cache)
 
-                self.out.warn("Migration: export_source cache new layout")
-                migrate_c_src_export_source(self.cache, self.out)
+        if old_version < Version("1.14.2"):
+            _migrate_full_metadata(self.cache, self.out)
 
 
 def _get_refs(cache):
@@ -159,6 +162,50 @@ def _get_prefs(layout):
     return [PackageReference(layout.ref, s) for s in folders]
 
 
+def _migrate_full_metadata(cache, out):
+    # Fix for https://github.com/conan-io/conan/issues/4898
+    out.warn("Running a full revision metadata migration")
+    refs = _get_refs(cache)
+    for ref in refs:
+        try:
+            base_folder = os.path.normpath(os.path.join(cache.store, ref.dir_repr()))
+            layout = PackageCacheLayout(base_folder=base_folder, ref=ref, short_paths=None,
+                                        no_lock=True)
+            with layout.update_metadata() as metadata:
+                # Updating the RREV
+                if metadata.recipe.revision is None:
+                    out.warn("Package %s metadata had recipe revision None, migrating" % str(ref))
+                    folder = layout.export()
+                    try:
+                        manifest = FileTreeManifest.load(folder)
+                        rrev = manifest.summary_hash
+                    except Exception:
+                        rrev = DEFAULT_REVISION_V1
+                    metadata.recipe.revision = rrev
+
+                prefs = _get_prefs(layout)
+                existing_ids = [pref.id for pref in prefs]
+                for pkg_id in list(metadata.packages.keys()):
+                    if pkg_id not in existing_ids:
+                        out.warn("Package %s metadata had stalled package information %s, removing"
+                                 % (str(ref), pkg_id))
+                        del metadata.packages[pkg_id]
+                # UPDATING PREVS
+                for pref in prefs:
+                    try:
+                        pmanifest = FileTreeManifest.load(layout.package(pref))
+                        prev = pmanifest.summary_hash
+                    except Exception:
+                        prev = DEFAULT_REVISION_V1
+                    metadata.packages[pref.id].revision = prev
+                    metadata.packages[pref.id].recipe_revision = metadata.recipe.revision
+
+        except Exception as e:
+            raise ConanException("Something went wrong while migrating metadata.json files "
+                                 "in the cache, please try to fix the issue or wipe the cache: {}"
+                                 ":{}".format(ref, e))
+
+
 def _migrate_create_metadata(cache, out):
     out.warn("Migration: Generating missing metadata files")
     refs = _get_refs(cache)
@@ -174,7 +221,7 @@ def _migrate_create_metadata(cache, out):
             try:
                 manifest = FileTreeManifest.load(folder)
                 rrev = manifest.summary_hash
-            except:
+            except Exception:
                 rrev = DEFAULT_REVISION_V1
             metadata_path = os.path.join(layout.conan(), PACKAGE_METADATA)
             if not os.path.exists(metadata_path):
@@ -186,7 +233,7 @@ def _migrate_create_metadata(cache, out):
                     try:
                         pmanifest = FileTreeManifest.load(layout.package(pref))
                         prev = pmanifest.summary_hash
-                    except:
+                    except Exception:
                         prev = DEFAULT_REVISION_V1
                     metadata.packages[pref.id].revision = prev
                     metadata.packages[pref.id].recipe_revision = metadata.recipe.revision


Test patch:
diff --git a/conans/test/functional/old/test_migrations.py b/conans/test/functional/old/test_migrations.py
--- a/conans/test/functional/old/test_migrations.py
+++ b/conans/test/functional/old/test_migrations.py
@@ -1,3 +1,4 @@
+import json
 import os
 import unittest
 
@@ -7,6 +8,8 @@
 from conans.client.migrations import migrate_plugins_to_hooks, migrate_to_default_profile
 from conans.client.output import ConanOutput
 from conans.migrations import CONAN_VERSION
+from conans.model.ref import ConanFileReference
+from conans.test.utils.conanfile import TestConanFile
 from conans.test.utils.test_files import temp_folder
 from conans.test.utils.tools import TestClient
 from conans.util.files import load, save
@@ -14,6 +17,42 @@
 
 class TestMigrations(unittest.TestCase):
 
+    def test_migrate_revision_metadata(self):
+        # https://github.com/conan-io/conan/issues/4898
+        client = TestClient()
+        client.save({"conanfile.py": str(TestConanFile("Hello", "0.1"))})
+        client.run("create . user/testing")
+        ref = ConanFileReference.loads("Hello/0.1@user/testing")
+        layout1 = client.cache.package_layout(ref)
+        metadata = json.loads(load(layout1.package_metadata()))
+        metadata["recipe"]["revision"] = None
+        metadata["packages"]["WRONG"] = {"revision": ""}
+        metadata["packages"]["5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"]["revision"] = None
+        metadata["packages"]["5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"]["recipe_revision"] = None
+        save(layout1.package_metadata(), json.dumps(metadata))
+
+        client.run("create . user/stable")
+        ref2 = ConanFileReference.loads("Hello/0.1@user/stable")
+        layout2 = client.cache.package_layout(ref2)
+        metadata = json.loads(load(layout2.package_metadata()))
+        metadata["recipe"]["revision"] = "Other"
+        save(layout2.package_metadata(), json.dumps(metadata))
+
+        version_file = os.path.join(client.cache.conan_folder, CONAN_VERSION)
+        save(version_file, "1.14.1")
+        client.run("search")  # This will fire a migration
+
+        metadata_ref1 = client.cache.package_layout(ref).load_metadata()
+        self.assertEqual(metadata_ref1.recipe.revision, "2e48797069e65568befd81854aa8aaf0")
+        pkg_metadata = metadata_ref1.packages["5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"]
+        self.assertEqual(pkg_metadata.recipe_revision, "2e48797069e65568befd81854aa8aaf0")
+        self.assertEqual(pkg_metadata.revision, "38d05b42210ca2becb43a05f9265abbe")
+        self.assertEqual(list(metadata_ref1.packages.keys()),
+                         ["5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"])
+
+        metadata_ref2 = client.cache.package_layout(ref2).load_metadata()
+        self.assertEqual(metadata_ref2.recipe.revision, "Other")
+
     def test_migrate_config_install(self):
         client = TestClient()
         client.run('config set general.config_install="url, http:/fake.url, None, None"')