Repository: conan-io/conan
Pull request: fix migration lru
Changelog: Bugfix: Fix the 2.0.14 migration to add LRU data to the cache when ``storage_path`` conf is defined.
Docs: Omit

Close https://github.com/conan-io/conan/pull/15130
Close https://github.com/conan-io/conan/issues/15129

Problem statement:
2.0.14's cache migrations do not respect core.cache:storage_path
### Environment details

* Operating System+version: Mac OS 13.4.1
* Compiler+version: n/a
* Conan version: 2.0.14
* Python version: 3.10.8


### Steps to reproduce

* install conan 2.0.13

* set `core.cache:storage_path` to something  other than `~/.conan2/p`.  remove `~/.conan2/p`

* upgrade to conan 2.0.14

* try to use conan, it will fail with "Could not complete the 2.0.14 DB migration"

### Logs

```
➜  conan  list \*    
WARN: Upgrade cache from Conan version '2.0.13'
WARN: Running 2.0.14 Cache DB migration to add LRU column
ERROR: unable to open database file
```

Hints from issue discussion:

Commit message: fix back migration

Patch to review:
diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py
index 22e30fd53b9..04b21e7bc18 100644
--- a/conan/api/subapi/config.py
+++ b/conan/api/subapi/config.py
@@ -43,28 +43,33 @@ def global_conf(self):
         to the profile ones and passed to the conanfiles.conf, which can be passed to collaborators
         """
         if self._new_config is None:
-            self._new_config = ConfDefinition()
             cache_folder = self.conan_api.cache_folder
-            home_paths = HomePaths(cache_folder)
-            global_conf_path = home_paths.global_conf_path
-            if os.path.exists(global_conf_path):
-                text = load(global_conf_path)
-                distro = None
-                if platform.system() in ["Linux", "FreeBSD"]:
-                    import distro
-                template = Environment(loader=FileSystemLoader(cache_folder)).from_string(text)
-                content = template.render({"platform": platform, "os": os, "distro": distro,
-                                           "conan_version": conan_version,
-                                           "conan_home_folder": cache_folder,
-                                           "detect_api": detect_api})
-                self._new_config.loads(content)
-            else:  # creation of a blank global.conf file for user convenience
-                default_global_conf = textwrap.dedent("""\
-                    # Core configuration (type 'conan config list' to list possible values)
-                    # e.g, for CI systems, to raise if user input would block
-                    # core:non_interactive = True
-                    # some tools.xxx config also possible, though generally better in profiles
-                    # tools.android:ndk_path = my/path/to/android/ndk
-                    """)
-                save(global_conf_path, default_global_conf)
+            self._new_config = self.load_config(cache_folder)
         return self._new_config
+
+    @staticmethod
+    def load_config(home_folder):
+        home_paths = HomePaths(home_folder)
+        global_conf_path = home_paths.global_conf_path
+        new_config = ConfDefinition()
+        if os.path.exists(global_conf_path):
+            text = load(global_conf_path)
+            distro = None
+            if platform.system() in ["Linux", "FreeBSD"]:
+                import distro
+            template = Environment(loader=FileSystemLoader(home_folder)).from_string(text)
+            content = template.render({"platform": platform, "os": os, "distro": distro,
+                                       "conan_version": conan_version,
+                                       "conan_home_folder": home_folder,
+                                       "detect_api": detect_api})
+            new_config.loads(content)
+        else:  # creation of a blank global.conf file for user convenience
+            default_global_conf = textwrap.dedent("""\
+                # Core configuration (type 'conan config list' to list possible values)
+                # e.g, for CI systems, to raise if user input would block
+                # core:non_interactive = True
+                # some tools.xxx config also possible, though generally better in profiles
+                # tools.android:ndk_path = my/path/to/android/ndk
+                """)
+            save(global_conf_path, default_global_conf)
+        return new_config
diff --git a/conans/client/migrations.py b/conans/client/migrations.py
index f7ea7af9008..a66675bac80 100644
--- a/conans/client/migrations.py
+++ b/conans/client/migrations.py
@@ -3,6 +3,7 @@
 import textwrap
 
 from conan.api.output import ConanOutput
+from conan.api.subapi.config import ConfigAPI
 from conans.migrations import Migrator
 from conans.util.dates import timestamp_now
 from conans.util.files import load, save
@@ -59,7 +60,9 @@ def _apply_migrations(self, old_version):
 def _migrate_pkg_db_lru(cache_folder, old_version):
     ConanOutput().warning(f"Upgrade cache from Conan version '{old_version}'")
     ConanOutput().warning("Running 2.0.14 Cache DB migration to add LRU column")
-    db_filename = os.path.join(cache_folder, 'p', 'cache.sqlite3')
+    config = ConfigAPI.load_config(cache_folder)
+    storage = config.get("core.cache:storage_path") or os.path.join(cache_folder, "p")
+    db_filename = os.path.join(storage, 'cache.sqlite3')
     connection = sqlite3.connect(db_filename, isolation_level=None,
                                  timeout=1, check_same_thread=False)
     try:
@@ -75,8 +78,21 @@ def _migrate_pkg_db_lru(cache_folder, old_version):
         undo_lru = textwrap.dedent("""\
             import os
             import sqlite3
+
             def migrate(cache_folder):
-                db = os.path.join(cache_folder, 'p', 'cache.sqlite3')
+                config = os.path.join(cache_folder, "global.conf")
+                global_conf = open(config, "r").read() if os.path.isfile(config) else ""
+                storage = os.path.join(cache_folder, "p")
+                for line in global_conf.splitlines():
+                    tokens = line.split("=")
+                    if len(tokens) != 2:
+                        continue
+                    conf, value = tokens
+                    if conf.strip() == "core.cache:storage_path":
+                        storage = value.strip()
+                        break
+
+                db = os.path.join(storage, 'cache.sqlite3')
                 connection = sqlite3.connect(db, isolation_level=None, timeout=1,
                                              check_same_thread=False)
                 rec_cols = 'reference, rrev, path, timestamp'
diff --git a/conans/test/integration/test_migrations.py b/conans/test/integration/test_migrations.py
index 363b30ba0c1..efa75434e5d 100644
--- a/conans/test/integration/test_migrations.py
+++ b/conans/test/integration/test_migrations.py
@@ -5,12 +5,14 @@
 import pytest
 
 from conan import conan_version
+from conans.test.assets.genconanfile import GenConanfile
+from conans.test.utils.test_files import temp_folder
 from conans.test.utils.tools import TestClient
 from conans.util.files import save, load
 
 
-def _drop_lru_column(cache_folder):
-    db = os.path.join(cache_folder, 'p', 'cache.sqlite3')
+def _drop_lru_column(db_folder):
+    db = os.path.join(db_folder, "cache.sqlite3")
     connection = sqlite3.connect(db, isolation_level=None, timeout=1, check_same_thread=False)
     rec_cols = 'reference, rrev, path, timestamp'
     pkg_cols = 'reference, rrev, pkgid, prev, path, timestamp, build_id'
@@ -39,7 +41,8 @@ def test_migration_profile_checker_plugin(plugin_path, string_replace, new_strin
     # Let's change the version
     version_txt_file_path = os.path.join(t.cache_folder, "version.txt")
     save(version_txt_file_path, "1.0.0")
-    _drop_lru_column(t.cache_folder)
+    db = os.path.join(t.cache_folder, 'p')
+    _drop_lru_column(db)
 
     # Do a modification to the profile plugin without changing the comment
     contents = contents.replace(string_replace, new_string)
@@ -70,7 +73,8 @@ def test_migration_profile_checker_plugin(plugin_path, string_replace, new_strin
     # Let's change the version
     version_txt_file_path2 = os.path.join(t2.cache_folder, "version.txt")
     save(version_txt_file_path2, "1.0.0")
-    _drop_lru_column(t2.cache_folder)
+    db = os.path.join(t2.cache_folder, 'p')
+    _drop_lru_column(db)
     # Trigger the migrations
     t2.run("-v")
     assert "WARN: Running 2.0.14 Cache DB migration to add LRU column" in t2.out
@@ -82,6 +86,24 @@ def test_migration_profile_checker_plugin(plugin_path, string_replace, new_strin
     assert new_string in contents
 
 
+def test_migration_db_lru():
+    t = TestClient()
+    storage = temp_folder()
+    save(t.cache.global_conf_path, f"core.cache:storage_path={storage}")
+    t.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+    t.run("create .")
+    # Any command generates the profile and compatibility plugin files
+    # Let's change the version
+    version_txt_file_path = os.path.join(t.cache_folder, "version.txt")
+    save(version_txt_file_path, "1.0.0")
+    _drop_lru_column(storage)
+
+    # Trigger the migrations
+    t.run("list *")
+    assert "WARN: Running 2.0.14 Cache DB migration to add LRU column" in t.out
+    assert "pkg/0.1" in t.out
+
+
 def test_back_migrations():
     t = TestClient()
 


Code review comments:
diff_hunk: @@ -75,8 +78,21 @@ def _migrate_pkg_db_lru(cache_folder, old_version):
         undo_lru = textwrap.dedent("""\
             import os
             import sqlite3
+
             def migrate(cache_folder):
-                db = os.path.join(cache_folder, 'p', 'cache.sqlite3')
+                config = os.path.join(cache_folder, "global.conf")
+                global_conf = open(config, "r").read() if os.path.isfile(config) else ""
+                storage = os.path.join(cache_folder, "p")
+                for line in global_conf.splitlines():
+                    tokens = line.split("=")
+                    if len(tokens) != 2:
+                        continue
+                    conf, value = tokens
path: conans/client/migrations.py
text: @user1:
shoudln't this use ConfDefinition to parse quotes and whatnot?

@author:
The truth is that I was trying to do a shortcut, to avoid having to check every change in the code in the previous versions, because the code has to be compatible with the old version, not that evident with all the work and changes since 2.0.0. But yeah, probably leveraging some of the previous code is more robust than this, I'll try to find some common base.

@author:
Done