Repository: mvt-project/mvt (language: python)

Issue:
2 Filesystem tests fail on macOS only
Only impacts development AFAICT. Would be good to track down a root cause.

```
FAILED tests/ios_fs/test_filesystem.py::TestFilesystem::test_filesystem - AssertionError: assert 17 == 15
FAILED tests/ios_fs/test_filesystem.py::TestFilesystem::test_detection - AssertionError: assert 17 == 15
```

When the tests fail it leaves behind two artifacts on disk:

```
	tests/artifacts/ios_backup/64/64d0019cb3d46bfc8cce545a8ba54b93e7ea9347-shm
	tests/artifacts/ios_backup/64/64d0019cb3d46bfc8cce545a8ba54b93e7ea9347-wal
```


Discussion / hints:
Running only `pytest tests/ios_fs/test_filesystem.py` works fine - I think one of the other test modules must be leaving behind these artifacts, interfering with this module's tests.
I think there is an issue with the SQLite driver on OS X. It loads the SQLite databases. created the temporary write-ahead-log and keeps does around until the process finished IIRC.

https://github.com/groue/GRDB.swift/issues/739#issuecomment-624483632

Fix patch:
diff --git a/mvt/ios/modules/backup/manifest.py b/mvt/ios/modules/backup/manifest.py
index 7f6c6e423..6dac3741c 100644
--- a/mvt/ios/modules/backup/manifest.py
+++ b/mvt/ios/modules/backup/manifest.py
@@ -8,7 +8,6 @@
 import logging
 import os
 import plistlib
-import sqlite3
 from typing import Optional
 
 from mvt.common.module import DatabaseNotFoundError
@@ -124,7 +123,7 @@ def run(self) -> None:
 
         self.log.info("Found Manifest.db database at path: %s", manifest_db_path)
 
-        conn = sqlite3.connect(manifest_db_path)
+        conn = self._open_sqlite_db(manifest_db_path)
         cur = conn.cursor()
 
         cur.execute("SELECT * FROM Files;")
diff --git a/mvt/ios/modules/base.py b/mvt/ios/modules/base.py
index ea44e16c0..635d9ad0b 100644
--- a/mvt/ios/modules/base.py
+++ b/mvt/ios/modules/base.py
@@ -49,7 +49,7 @@ def _recover_sqlite_db_if_needed(
         """
         # TODO: Find a better solution.
         if not forced:
-            conn = sqlite3.connect(file_path)
+            conn = self._open_sqlite_db(file_path)
             cur = conn.cursor()
 
             try:
@@ -91,6 +91,9 @@ def _recover_sqlite_db_if_needed(
 
         self.log.info("Database at path %s recovered successfully!", file_path)
 
+    def _open_sqlite_db(self, file_path: str) -> sqlite3.Connection:
+        return sqlite3.connect(f"file:{file_path}?immutable=1")
+
     def _get_backup_files_from_manifest(
         self, relative_path: Optional[str] = None, domain: Optional[str] = None
     ) -> Iterator[dict]:
@@ -109,7 +112,7 @@ def _get_backup_files_from_manifest(
         base_sql = "SELECT fileID, domain, relativePath FROM Files WHERE "
 
         try:
-            conn = sqlite3.connect(manifest_db_path)
+            conn = self._open_sqlite_db(manifest_db_path)
             cur = conn.cursor()
             if relative_path and domain:
                 cur.execute(
diff --git a/mvt/ios/modules/fs/analytics.py b/mvt/ios/modules/fs/analytics.py
index 37ee562bd..5165f666d 100644
--- a/mvt/ios/modules/fs/analytics.py
+++ b/mvt/ios/modules/fs/analytics.py
@@ -85,7 +85,7 @@ def check_indicators(self) -> None:
     def _extract_analytics_data(self):
         artifact = self.file_path.split("/")[-1]
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
 
         try:
diff --git a/mvt/ios/modules/fs/cache_files.py b/mvt/ios/modules/fs/cache_files.py
index 2f27cb8a3..5a91efcac 100644
--- a/mvt/ios/modules/fs/cache_files.py
+++ b/mvt/ios/modules/fs/cache_files.py
@@ -64,7 +64,7 @@ def check_indicators(self) -> None:
     def _process_cache_file(self, file_path):
         self.log.info("Processing cache file at path: %s", file_path)
 
-        conn = sqlite3.connect(file_path)
+        conn = self._open_sqlite_db(file_path)
         cur = conn.cursor()
 
         try:
diff --git a/mvt/ios/modules/fs/safari_favicon.py b/mvt/ios/modules/fs/safari_favicon.py
index 979ee1552..aadd19131 100644
--- a/mvt/ios/modules/fs/safari_favicon.py
+++ b/mvt/ios/modules/fs/safari_favicon.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_mactime_to_iso
@@ -61,7 +60,7 @@ def check_indicators(self) -> None:
                 self.detected.append(result)
 
     def _process_favicon_db(self, file_path):
-        conn = sqlite3.connect(file_path)
+        conn = self._open_sqlite_db(file_path)
 
         # Fetch valid icon cache.
         cur = conn.cursor()
diff --git a/mvt/ios/modules/mixed/calendar.py b/mvt/ios/modules/mixed/calendar.py
index 395b47b5e..bfd1fc79a 100644
--- a/mvt/ios/modules/mixed/calendar.py
+++ b/mvt/ios/modules/mixed/calendar.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_mactime_to_iso
@@ -82,7 +81,7 @@ def _parse_calendar_db(self):
         """
         Parse the calendar database
         """
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
 
         cur.execute(
diff --git a/mvt/ios/modules/mixed/calls.py b/mvt/ios/modules/mixed/calls.py
index 03280e867..32f44a55e 100644
--- a/mvt/ios/modules/mixed/calls.py
+++ b/mvt/ios/modules/mixed/calls.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_mactime_to_iso
@@ -53,7 +52,7 @@ def run(self) -> None:
         )
         self.log.info("Found Calls database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         cur.execute(
             """
diff --git a/mvt/ios/modules/mixed/chrome_favicon.py b/mvt/ios/modules/mixed/chrome_favicon.py
index 5e28e8ece..addfe328e 100644
--- a/mvt/ios/modules/mixed/chrome_favicon.py
+++ b/mvt/ios/modules/mixed/chrome_favicon.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_chrometime_to_datetime, convert_datetime_to_iso
@@ -66,7 +65,7 @@ def run(self) -> None:
         )
         self.log.info("Found Chrome favicon cache database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
 
         # Fetch icon cache
         cur = conn.cursor()
diff --git a/mvt/ios/modules/mixed/chrome_history.py b/mvt/ios/modules/mixed/chrome_history.py
index 2b314c424..e934a6c3d 100644
--- a/mvt/ios/modules/mixed/chrome_history.py
+++ b/mvt/ios/modules/mixed/chrome_history.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_chrometime_to_datetime, convert_datetime_to_iso
@@ -67,7 +66,7 @@ def run(self) -> None:
         )
         self.log.info("Found Chrome history database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         cur.execute(
             """
diff --git a/mvt/ios/modules/mixed/contacts.py b/mvt/ios/modules/mixed/contacts.py
index df3109139..5f842c646 100644
--- a/mvt/ios/modules/mixed/contacts.py
+++ b/mvt/ios/modules/mixed/contacts.py
@@ -44,7 +44,7 @@ def run(self) -> None:
         )
         self.log.info("Found Contacts database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         try:
             cur.execute(
diff --git a/mvt/ios/modules/mixed/firefox_favicon.py b/mvt/ios/modules/mixed/firefox_favicon.py
index 4633d0116..173e2e7da 100644
--- a/mvt/ios/modules/mixed/firefox_favicon.py
+++ b/mvt/ios/modules/mixed/firefox_favicon.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_unix_to_iso
@@ -68,7 +67,7 @@ def run(self) -> None:
         )
         self.log.info("Found Firefox favicon database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         cur.execute(
             """
diff --git a/mvt/ios/modules/mixed/firefox_history.py b/mvt/ios/modules/mixed/firefox_history.py
index e720580b3..851096715 100644
--- a/mvt/ios/modules/mixed/firefox_history.py
+++ b/mvt/ios/modules/mixed/firefox_history.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import convert_unix_to_iso
@@ -68,7 +67,7 @@ def run(self) -> None:
         )
         self.log.info("Found Firefox history database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         cur.execute(
             """
diff --git a/mvt/ios/modules/mixed/interactionc.py b/mvt/ios/modules/mixed/interactionc.py
index cc31d2060..744decddb 100644
--- a/mvt/ios/modules/mixed/interactionc.py
+++ b/mvt/ios/modules/mixed/interactionc.py
@@ -280,7 +280,7 @@ def run(self) -> None:
         )
         self.log.info("Found InteractionC database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
 
         try:
diff --git a/mvt/ios/modules/mixed/safari_browserstate.py b/mvt/ios/modules/mixed/safari_browserstate.py
index 3584a5b6f..e0242ab8f 100644
--- a/mvt/ios/modules/mixed/safari_browserstate.py
+++ b/mvt/ios/modules/mixed/safari_browserstate.py
@@ -76,7 +76,7 @@ def check_indicators(self) -> None:
 
     def _process_browser_state_db(self, db_path):
         self._recover_sqlite_db_if_needed(db_path)
-        conn = sqlite3.connect(db_path)
+        conn = self._open_sqlite_db(db_path)
 
         cur = conn.cursor()
         try:
diff --git a/mvt/ios/modules/mixed/safari_history.py b/mvt/ios/modules/mixed/safari_history.py
index 200b463bf..cfa7361e6 100644
--- a/mvt/ios/modules/mixed/safari_history.py
+++ b/mvt/ios/modules/mixed/safari_history.py
@@ -5,7 +5,6 @@
 
 import logging
 import os
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.url import URL
@@ -115,7 +114,7 @@ def check_indicators(self) -> None:
 
     def _process_history_db(self, history_path):
         self._recover_sqlite_db_if_needed(history_path)
-        conn = sqlite3.connect(history_path)
+        conn = self._open_sqlite_db(history_path)
         cur = conn.cursor()
         cur.execute(
             """
diff --git a/mvt/ios/modules/mixed/shortcuts.py b/mvt/ios/modules/mixed/shortcuts.py
index 25dac6019..47f9386d3 100644
--- a/mvt/ios/modules/mixed/shortcuts.py
+++ b/mvt/ios/modules/mixed/shortcuts.py
@@ -83,7 +83,7 @@ def run(self) -> None:
         )
         self.log.info("Found Shortcuts database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         conn.text_factory = bytes
         cur = conn.cursor()
         try:
diff --git a/mvt/ios/modules/mixed/sms.py b/mvt/ios/modules/mixed/sms.py
index 0f0cee740..c538f1b46 100644
--- a/mvt/ios/modules/mixed/sms.py
+++ b/mvt/ios/modules/mixed/sms.py
@@ -86,7 +86,7 @@ def run(self) -> None:
         self.log.info("Found SMS database at path: %s", self.file_path)
 
         try:
-            conn = sqlite3.connect(self.file_path)
+            conn = self._open_sqlite_db(self.file_path)
             cur = conn.cursor()
             cur.execute(
                 """
@@ -103,7 +103,7 @@ def run(self) -> None:
             conn.close()
             if "database disk image is malformed" in str(exc):
                 self._recover_sqlite_db_if_needed(self.file_path, forced=True)
-                conn = sqlite3.connect(self.file_path)
+                conn = self._open_sqlite_db(self.file_path)
                 cur = conn.cursor()
                 cur.execute(
                     """
diff --git a/mvt/ios/modules/mixed/sms_attachments.py b/mvt/ios/modules/mixed/sms_attachments.py
index 0eb7acb7f..3e8cf4d3c 100644
--- a/mvt/ios/modules/mixed/sms_attachments.py
+++ b/mvt/ios/modules/mixed/sms_attachments.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from base64 import b64encode
 from typing import Optional, Union
 
@@ -72,7 +71,7 @@ def run(self) -> None:
         self._find_ios_database(backup_ids=SMS_BACKUP_IDS, root_paths=SMS_ROOT_PATHS)
         self.log.info("Found SMS database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         cur.execute(
             """
diff --git a/mvt/ios/modules/mixed/tcc.py b/mvt/ios/modules/mixed/tcc.py
index cac0ed4bb..461e5b310 100644
--- a/mvt/ios/modules/mixed/tcc.py
+++ b/mvt/ios/modules/mixed/tcc.py
@@ -95,7 +95,7 @@ def check_indicators(self) -> None:
                 self.detected.append(result)
 
     def process_db(self, file_path):
-        conn = sqlite3.connect(file_path)
+        conn = self._open_sqlite_db(file_path)
         cur = conn.cursor()
         db_version = "v3"
         try:
diff --git a/mvt/ios/modules/mixed/webkit_resource_load_statistics.py b/mvt/ios/modules/mixed/webkit_resource_load_statistics.py
index cd7b8c16b..f5beec6a6 100644
--- a/mvt/ios/modules/mixed/webkit_resource_load_statistics.py
+++ b/mvt/ios/modules/mixed/webkit_resource_load_statistics.py
@@ -73,7 +73,7 @@ def _process_observations_db(self, db_path: str, domain: str, path: str) -> None
 
         self._recover_sqlite_db_if_needed(db_path)
 
-        conn = sqlite3.connect(db_path)
+        conn = self._open_sqlite_db(db_path)
         cur = conn.cursor()
 
         try:
diff --git a/mvt/ios/modules/mixed/whatsapp.py b/mvt/ios/modules/mixed/whatsapp.py
index f44506876..7bf99002b 100644
--- a/mvt/ios/modules/mixed/whatsapp.py
+++ b/mvt/ios/modules/mixed/whatsapp.py
@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from typing import Optional, Union
 
 from mvt.common.utils import check_for_links, convert_mactime_to_iso
@@ -69,7 +68,7 @@ def run(self) -> None:
         )
         self.log.info("Found WhatsApp database at path: %s", self.file_path)
 
-        conn = sqlite3.connect(self.file_path)
+        conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
 
         # Query all messages and join tables which can contain media attachments


Test patch:
diff --git a/tests/ios_fs/test_filesystem.py b/tests/ios_fs/test_filesystem.py
index 9dad8b339..4e6cbb6ef 100644
--- a/tests/ios_fs/test_filesystem.py
+++ b/tests/ios_fs/test_filesystem.py
@@ -4,31 +4,23 @@
 #   https://license.mvt.re/1.1/
 import logging
 
-import pytest
 
 from mvt.common.indicators import Indicators
 from mvt.common.module import run_module
 from mvt.ios.modules.fs.filesystem import Filesystem
 
-from ..utils import delete_tmp_db_files, get_ios_backup_folder
-
-
-@pytest.fixture()
-def cleanup_tmp_artifacts():
-    ios_backup_folder = get_ios_backup_folder()
-    delete_tmp_db_files(ios_backup_folder)
-    return
+from ..utils import get_ios_backup_folder
 
 
 class TestFilesystem:
-    def test_filesystem(self, cleanup_tmp_artifacts):
+    def test_filesystem(self):
         m = Filesystem(target_path=get_ios_backup_folder())
         run_module(m)
         assert len(m.results) == 15
         assert len(m.timeline) == 15
         assert len(m.detected) == 0
 
-    def test_detection(self, indicator_file, cleanup_tmp_artifacts):
+    def test_detection(self, indicator_file):
         m = Filesystem(target_path=get_ios_backup_folder())
         ind = Indicators(log=logging.getLogger())
         ind.parse_stix2(indicator_file)