Repository: wkeeling/selenium-wire

Issue:
_pickle.UnpicklingError: pickle data was truncated
When itterating over driver requests I get this error:
`_pickle.UnpicklingError: pickle data was truncated`
Sometimes it doesn't throw it, but most oftenly it does. It's really weird.
edit:
Alright so I figured that if you wrap it a try: except: block and just try again it works so thats a temporary fix for now

Discussion:

Patch:
diff --git a/seleniumwire/storage.py b/seleniumwire/storage.py
index d569544..6d1cd66 100644
--- a/seleniumwire/storage.py
+++ b/seleniumwire/storage.py
@@ -171,15 +171,23 @@ class RequestStorage:
 
         for indexed_request in index:
             request = self._load_request(indexed_request.id)
-            loaded.append(request)
+
+            if request is not None:
+                loaded.append(request)
 
         return loaded
 
-    def _load_request(self, request_id: str) -> Request:
+    def _load_request(self, request_id: str) -> Optional[Request]:
         request_dir = self._get_request_dir(request_id)
 
         with open(os.path.join(request_dir, 'request'), 'rb') as req:
-            request = pickle.load(req)
+            try:
+                request = pickle.load(req)
+            except Exception:
+                # Errors may sometimes occur with unpickling - e.g.
+                # sometimes data hasn't been fully flushed to disk
+                # by the OS by the time we come to unpickle it.
+                return None
 
             ws_messages = self._ws_messages.get(request.id)
 


Test patch:
diff --git a/tests/seleniumwire/test_storage.py b/tests/seleniumwire/test_storage.py
index 24434c4..a31ab9b 100644
--- a/tests/seleniumwire/test_storage.py
+++ b/tests/seleniumwire/test_storage.py
@@ -175,6 +175,19 @@ class RequestStorageTest(TestCase):
         self.assertIsNone(requests[0].response)
         self.assertIsNone(requests[1].response)
 
+    @patch('seleniumwire.storage.pickle')
+    def test_load_requests_unpickle_error(self, mock_pickle):
+        request_1 = self._create_request()
+        request_2 = self._create_request()
+        mock_pickle.load.side_effect = [Exception, request_2]
+        self.storage.save_request(request_1)
+        self.storage.save_request(request_2)
+
+        requests = self.storage.load_requests()
+
+        self.assertEqual(1, len(requests))
+        self.assertEqual(request_2.id, requests[0].id)
+
     def test_iter_requests(self):
         request_1 = self._create_request()
         request_2 = self._create_request()