Repository: glasgow-ipl/ietfdata (language: python)

Issue:
Changes for the Example notebook
These are changes that are needed in the "ietfdata" code to make the analysis notebook work as intended. They mainly consist of minor additions/changes to error handling of the borderline cases and introducing some imap keepalives during download (also I changed policy to be policy.default in message_from_bytes and message_from_binary file).
Changes for the Example notebook
These are changes that are needed in the "ietfdata" code to make the analysis notebook work as intended. They mainly consist of minor additions/changes to error handling of the borderline cases and introducing some imap keepalives during download (also I changed policy to be policy.default in message_from_bytes and message_from_binary file).


Discussion / hints:

Fix patch:
diff --git a/Pipfile b/Pipfile
index e14f1dd..8e14471 100644
--- a/Pipfile
+++ b/Pipfile
@@ -18,6 +18,7 @@ pavlova = "*"
 imapclient = "*"
 python-dateutil = "*"
 pymongo = "*"
+pandas-stubs = "*"
 
 [requires]
 python_version = "3.9"
diff --git a/Pipfile.lock b/Pipfile.lock
index 18e8db5..a937155 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "e77612627fab544dd9c7a6be5199e4ba8db2ad9834c3269ef3d60aea5f990ee1"
+            "sha256": "2535c72a2405af315fd2ce460dd4a12e8a2793e87e771b9bbf0ca18b7e96e220"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -55,6 +55,14 @@
             "index": "pypi",
             "version": "==2.2.0"
         },
+        "pandas-stubs": {
+            "hashes": [
+                "sha256:4b16d8373e9a9c030abe7e72d27551ca8536ac8efa4b25ad76d710ca9e905ace",
+                "sha256:56ab49a0e414bc5b2aa20efb4d906ddefa8270a14309fd743e433456bd89b3b9"
+            ],
+            "index": "pypi",
+            "version": "==1.1.0.6"
+        },
         "pavlova": {
             "hashes": [
                 "sha256:1a5acb53c288dc50b2eeb594dfc07a3af3bb8b0a7066e195d257d05c7c2ee1c8",
@@ -65,73 +73,73 @@
         },
         "pymongo": {
             "hashes": [
-                "sha256:0384d76b409278ddb34ac19cdc4664511685959bf719adbdc051875ded4689aa",
-                "sha256:05e2bda928a3a6bc6ddff9e5a8579d41928b75d7417b18f9a67c82bb52150ac6",
-                "sha256:152e4ac3158b776135d8fce28d2ac06e682b885fcbe86690d66465f262ab244e",
-                "sha256:180511abfef70feb022360b35f4863dd68e08334197089201d5c52208de9ca2e",
-                "sha256:19d52c60dc37520385f538d6d1a4c40bc398e0885f4ed6a36ce10b631dab2852",
-                "sha256:1d559a76ae87143ad96c2ecd6fdd38e691721e175df7ced3fcdc681b4638bca1",
-                "sha256:210ec4a058480b9c3869082e52b66d80c4a48eda9682d7a569a1a5a48100ea54",
-                "sha256:2163d736d6f62b20753be5da3dc07a188420b355f057fcbb3075b05ee6227b2f",
-                "sha256:22ee2c94fee1e391735be63aa1c9af4c69fdcb325ae9e5e4ddff770248ef60a6",
-                "sha256:28633868be21a187702a8613913e13d1987d831529358c29fc6f6670413df040",
-                "sha256:29390c39ca873737689a0749c9c3257aad96b323439b11279fbc0ba8626ec9c5",
-                "sha256:2aeb108da1ed8e066800fb447ba5ae89d560e6773d228398a87825ac3630452d",
-                "sha256:322f6cc7bf23a264151ebc5229a92600c4b55ac83c83c91c9bab1ec92c888a8d",
-                "sha256:34c15f5798f23488e509eae82fbf749c3d17db74379a88c07c869ece1aa806b9",
-                "sha256:3873866534b6527e6863e742eb23ea2a539e3c7ee00ad3f9bec9da27dbaaff6f",
-                "sha256:3dbc67754882d740f17809342892f0b24398770bd99d48c5cb5ba89f5f5dee4e",
-                "sha256:413b18ac2222f5d961eb8d1c8dcca6c6ca176c8613636d8c13aa23abae7f7a21",
-                "sha256:42f9ec9d77358f557fe17cc15e796c4d4d492ede1a30cba3664822cae66e97c5",
-                "sha256:4ac387ac1be71b798d1c372a924f9c30352f30e684e06f086091297352698ac0",
-                "sha256:4ca92e15fcf02e02e7c24b448a16599b98c9d0e6a46cd85cc50804450ebf7245",
-                "sha256:4d959e929cec805c2bf391418b1121590b4e7d5cb00af7b1ba521443d45a0918",
-                "sha256:5091aacbdb667b418b751157f48f6daa17142c4f9063d58e5a64c90b2afbdf9a",
-                "sha256:5a03ae5ac85b04b2034a0689add9ff597b16d5e24066a87f6ab0e9fa67049156",
-                "sha256:5e1341276ce8b7752db9aeac6bbb0cbe82a3f6a6186866bf6b4906d8d328d50b",
-                "sha256:6043d251fac27ca04ff22ed8deb5ff7a43dc18e8a4a15b4c442d2a20fa313162",
-                "sha256:610d5cbbfd026e2f6d15665af51e048e49b68363fedece2ed318cc8fe080dd94",
-                "sha256:622a5157ffcd793d305387c1c9fb94185f496c8c9fd66dafb59de0807bc14ad7",
-                "sha256:65b67637f0a25ac9d25efb13c1578eb065870220ffa82f132c5b2d8e43ac39c3",
-                "sha256:66573c8c7808cce4f3b56c23cb7cad6c3d7f4c464b9016d35f5344ad743896d7",
-                "sha256:66b688fc139c6742057795510e3b12c4acbf90d11af1eff9689a41d9c84478d6",
-                "sha256:685b884fa41bd2913fd20af85866c4ff886b7cbb7e4833b918996aa5d45a04be",
-                "sha256:6a5834e392c97f19f36670e34bf9d346d733ad89ee0689a6419dd737dfa4308a",
-                "sha256:728313cc0d59d1a1a004f675607dcf5c711ced3f55e75d82b3f264fd758869f3",
-                "sha256:733e1cfffc4cd99848230e2999c8a86e284c6af6746482f8ad2ad554dce14e39",
-                "sha256:7814b2cf23aad23464859973c5cd2066ca2fd99e0b934acefbb0b728ac2525bf",
-                "sha256:7c77801620e5e75fb9c7abae235d3cc45d212a67efa98f4972eef63e736a8daa",
-                "sha256:7cd42c66d49ffb68dea065e1c8a4323e7ceab386e660fee9863d4fa227302ba9",
-                "sha256:7d2ae2f7c50adec20fde46a73465de31a6a6fbb4903240f8b7304549752ca7a1",
-                "sha256:7edff02e44dd0badd749d7342e40705a398d98c5d8f7570f57cff9568c2351fa",
-                "sha256:87981008d565f647142869d99915cc4760b7725858da3d39ecb2a606e23f36fd",
-                "sha256:92e2376ce3ca0e3e443b3c5c2bb5d584c7e59221edfb0035313c6306049ba55a",
-                "sha256:950710f7370613a6bfa2ccd842b488c5b8072e83fb6b7d45d99110bf44651d06",
-                "sha256:980527f4ccc6644855bb68056fe7835da6d06d37776a52df5bcc1882df57c3db",
-                "sha256:9fbffc5bad4df99a509783cbd449ed0d24fcd5a450c28e7756c8f20eda3d2aa5",
-                "sha256:a8b02e0119d6ee381a265d8d2450a38096f82916d895fed2dfd81d4c7a54d6e4",
-                "sha256:b17e627844d86031c77147c40bf992a6e1114025a460874deeda6500d0f34862",
-                "sha256:b1aa62903a2c5768b0001632efdea2e8da6c80abdd520c2e8a16001cc9affb23",
-                "sha256:b32e4eed2ef19a20dfb57698497a9bc54e74efb2e260c003e9056c145f130dc7",
-                "sha256:b44fa04720bbfd617b6aef036989c8c30435f11450c0a59136291d7b41ed647f",
-                "sha256:b4535d98df83abebb572035754fb3d4ad09ce7449375fa09fa9ede2dbc87b62b",
-                "sha256:bb6a5777bf558f444cd4883d617546182cfeff8f2d4acd885253f11a16740534",
-                "sha256:bc2eb67387b8376120a2be6cba9d23f9d6a6c3828e00fb0a64c55ad7b54116d1",
-                "sha256:bd351ceb2decd23d523fc50bad631ee9ae6e97e7cdc355ce5600fe310484f96e",
-                "sha256:bf70097bd497089f1baabf9cbb3ec4f69c022dc7a70c41ba9c238fa4d0fff7ab",
-                "sha256:c7fd18d4b7939408df9315fedbdb05e179760960a92b3752498e2fcd03f24c3d",
-                "sha256:cc359e408712faf9ea775f4c0ec8f2bfc843afe47747a657808d9595edd34d71",
-                "sha256:cd8fc35d4c0c717cc29b0cb894871555cb7137a081e179877ecc537e2607f0b9",
-                "sha256:daa44cefde19978af57ac1d50413cd86ebf2b497328e7a27832f5824bda47439",
-                "sha256:db5098587f58fbf8582d9bda2462762b367207246d3e19623782fb449c3c5fcc",
-                "sha256:db6fd53ef5f1914ad801830406440c3bfb701e38a607eda47c38adba267ba300",
-                "sha256:e1414599a97554d451e441afb362dbee1505e4550852c0068370d843757a3fe2",
-                "sha256:ee42a8f850143ae7c67ea09a183a6a4ad8d053e1dbd9a1134e21a7b5c1bc6c73",
-                "sha256:f23abcf6eca5859a2982beadfb5111f8c5e76e30ff99aaee3c1c327f814f9f10",
-                "sha256:f6748c447feeadda059719ef5ab1fb9d84bd370e205b20049a0e8b45ef4ad593"
+                "sha256:03be7ad107d252bb7325d4af6309fdd2c025d08854d35f0e7abc8bf048f4245e",
+                "sha256:071552b065e809d24c5653fcc14968cfd6fde4e279408640d5ac58e3353a3c5f",
+                "sha256:08b8723248730599c9803ae4c97b8f3f76c55219104303c88cb962a31e3bb5ee",
+                "sha256:08bda7b2c522ff9f1e554570da16298271ebb0c56ab9699446aacba249008988",
+                "sha256:0aaf4d44f1f819360f9432df538d54bbf850f18152f34e20337c01b828479171",
+                "sha256:0cabfc297f4cf921f15bc789a8fbfd7115eb9f813d3f47a74b609894bc66ab0d",
+                "sha256:13acf6164ead81c9fc2afa0e1ea6d6134352973ce2bb35496834fee057063c04",
+                "sha256:15b083d1b789b230e5ac284442d9ecb113c93f3785a6824f748befaab803b812",
+                "sha256:161fcd3281c42f644aa8dec7753cca2af03ce654e17d76da4f0dab34a12480ca",
+                "sha256:1a994a42f49dab5b6287e499be7d3d2751776486229980d8857ad53b8333d469",
+                "sha256:20d75ea11527331a2980ab04762a9d960bcfea9475c54bbeab777af880de61cd",
+                "sha256:225c61e08fe517aede7912937939e09adf086c8e6f7e40d4c85ad678c2c2aea3",
+                "sha256:3135dd574ef1286189f3f04a36c8b7a256376914f8cbbce66b94f13125ded858",
+                "sha256:3491c7de09e44eded16824cb58cf9b5cc1dc6f066a0bb7aa69929d02aa53b828",
+                "sha256:3551912f5c34d8dd7c32c6bb00ae04192af47f7b9f653608f107d19c1a21a194",
+                "sha256:38a7b5140a48fc91681cdb5cb95b7cd64640b43d19259fdd707fa9d5a715f2b2",
+                "sha256:3a3498a8326111221560e930f198b495ea6926937e249f475052ffc6893a6680",
+                "sha256:3bfc7689a1bacb9bcd2f2d5185d99507aa29f667a58dd8adaa43b5a348139e46",
+                "sha256:421d13523d11c57f57f257152bc4a6bb463aadf7a3918e9c96fefdd6be8dbfb8",
+                "sha256:424799c71ff435094e5fb823c40eebb4500f0e048133311e9c026467e8ccebac",
+                "sha256:474e21d0e07cd09679e357d1dac76e570dab86665e79a9d3354b10a279ac6fb3",
+                "sha256:4c7e8c8e1e1918dcf6a652ac4b9d87164587c26fd2ce5dd81e73a5ab3b3d492f",
+                "sha256:506a6dab4c7ffdcacdf0b8e70bd20eb2e77fa994519547c9d88d676400fcad58",
+                "sha256:510cd3bfabb63a07405b7b79fae63127e34c118b7531a2cbbafc7a24fd878594",
+                "sha256:517ba47ca04a55b1f50ee8df9fd97f6c37df5537d118fb2718952b8623860466",
+                "sha256:539d4cb1b16b57026999c53e5aab857fe706e70ae5310cc8c232479923f932e6",
+                "sha256:5c36428cc4f7fae56354db7f46677fd21222fc3cb1e8829549b851172033e043",
+                "sha256:5db59223ed1e634d842a053325f85f908359c6dac9c8ddce8ef145061fae7df8",
+                "sha256:5e606846c049ed40940524057bfdf1105af6066688c0e6a1a3ce2038589bae70",
+                "sha256:6060794aac9f7b0644b299f46a9c6cbc0bc470bd01572f4134df140afd41ded6",
+                "sha256:62c29bc36a6d9be68fe7b5aaf1e120b4aa66a958d1e146601fcd583eb12cae7b",
+                "sha256:73326b211e7410c8bd6a74500b1e3f392f39cf10862e243d00937e924f112c01",
+                "sha256:78f07961f4f214ea8e80be63cffd5cc158eb06cd922ffbf6c7155b11728f28f9",
+                "sha256:7c97554ea521f898753d9773891d0347ebfaddcc1dee2ad94850b163171bf1f1",
+                "sha256:8898f6699f740ca93a0879ed07d8e6db02d68af889d0ebb3d13ab017e6b1af1e",
+                "sha256:8a41fdc751dc4707a4fafb111c442411816a7c225ebb5cadb57599534b5d5372",
+                "sha256:8e0004b0393d72d76de94b4792a006cb960c1c65c7659930fbf9a81ce4341982",
+                "sha256:977b1d4f868986b4ba5d03c317fde4d3b66e687d74473130cd598e3103db34fa",
+                "sha256:9a4f6e0b01df820ba9ed0b4e618ca83a1c089e48d4f268d0e00dcd49893d4549",
+                "sha256:9b9298964389c180a063a9e8bac8a80ed42de11d04166b20249bfa0a489e0e0f",
+                "sha256:a08c8b322b671857c81f4c30cd3c8df2895fd3c0e9358714f39e0ef8fb327702",
+                "sha256:ad31f184dcd3271de26ab1f9c51574afb99e1b0e484ab1da3641256b723e4994",
+                "sha256:aff3656af2add93f290731a6b8930b23b35c0c09569150130a58192b3ec6fc61",
+                "sha256:b2f41261b648cf5dee425f37ff14f4ad151c2f24b827052b402637158fd056ef",
+                "sha256:b413117210fa6d92664c3d860571e8e8727c3e8f2ff197276c5d0cb365abd3ad",
+                "sha256:b7efc7e7049ef366777cfd35437c18a4166bb50a5606a1c840ee3b9624b54fc9",
+                "sha256:b8f94acd52e530a38f25e4d5bf7ddfdd4bea9193e718f58419def0d4406b58d3",
+                "sha256:d0a70151d7de8a3194cdc906bcc1a42e14594787c64b0c1c9c975e5a2af3e251",
+                "sha256:d360e5d5dd3d55bf5d1776964625018d85b937d1032bae1926dd52253decd0db",
+                "sha256:d4e62417e89b717a7bcd8576ac3108cd063225942cc91c5b37ff5465fdccd386",
+                "sha256:d65bac5f6724d9ea6f0b5a0f0e4952fbbf209adcf6b5583b54c54bd2fcd74dc0",
+                "sha256:e02beaab433fd1104b2804f909e694cfbdb6578020740a9051597adc1cd4e19f",
+                "sha256:e4b631688dfbdd61b5610e20b64b99d25771c6d52d9da73349342d2a0f11c46a",
+                "sha256:e4e9db78b71db2b1684ee4ecc3e32c4600f18cdf76e6b9ae03e338e52ee4b168",
+                "sha256:eb4d176394c37a76e8b0afe54b12d58614a67a60a7f8c0dd3a5afbb013c01092",
+                "sha256:f08665d3cc5abc2f770f472a9b5f720a9b3ab0b8b3bb97c7c1487515e5653d39",
+                "sha256:f3d851af3852f16ad4adc7ee054fd9c90a7a5063de94d815b7f6a88477b9f4c6",
+                "sha256:f4ba58157e8ae33ee86fadf9062c506e535afd904f07f9be32731f4410a23b7f",
+                "sha256:f664ed7613b8b18f0ce5696b146776266a038c19c5cd6efffa08ecc189b01b73",
+                "sha256:f947b359cc4769af8b49be7e37af01f05fcf15b401da2528021148e4a54426d1",
+                "sha256:fe4189846448df013cd9df11bba38ddf78043f8c290a9f06430732a7a8601cce",
+                "sha256:fea5cb1c63efe1399f0812532c7cf65458d38fd011be350bc5021dfcac39fba8",
+                "sha256:fedf0dee7a412ca6d1d6d92c158fe9cbaa8ea0cae90d268f9ccc0744de7a97d0",
+                "sha256:fffff7bfb6799a763d3742c59c6ee7ffadda21abed557637bc44ed1080876484"
             ],
             "index": "pypi",
-            "version": "==3.11.3"
+            "version": "==3.11.4"
         },
         "python-dateutil": {
             "hashes": [
@@ -204,11 +212,11 @@
         },
         "six": {
             "hashes": [
-                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
-                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.15.0"
+            "version": "==1.16.0"
         },
         "tzlocal": {
             "hashes": [
@@ -318,11 +326,11 @@
         },
         "docutils": {
             "hashes": [
-                "sha256:a71042bb7207c03d5647f280427f14bfbd1a65c9eb84f4b341d85fafb6bb4bdf",
-                "sha256:e2ffeea817964356ba4470efba7c2f42b6b0de0b04e66378507e3e2504bbff4c"
+                "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125",
+                "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==0.17"
+            "version": "==0.17.1"
         },
         "idna": {
             "hashes": [
@@ -334,11 +342,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:2ec0faae539743ae6aaa84b49a169670a465f7f5d64e6add98388cc29fd1f2f6",
-                "sha256:c9356b657de65c53744046fa8f7358afe0714a1af7d570c00c3835c2d724a7c1"
+                "sha256:8c501196e49fb9df5df43833bdb1e4328f64847763ec8a50703148b73784d581",
+                "sha256:d7eb1dea6d6a6086f8be21784cc9e3bcfa55872b52309bc5fad53a8ea444465d"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==3.10.1"
+            "version": "==4.0.1"
         },
         "keyring": {
             "hashes": [
@@ -400,81 +408,81 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94",
-                "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8"
+                "sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f",
+                "sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==2.8.1"
+            "version": "==2.9.0"
         },
         "pymongo": {
             "hashes": [
-                "sha256:0384d76b409278ddb34ac19cdc4664511685959bf719adbdc051875ded4689aa",
-                "sha256:05e2bda928a3a6bc6ddff9e5a8579d41928b75d7417b18f9a67c82bb52150ac6",
-                "sha256:152e4ac3158b776135d8fce28d2ac06e682b885fcbe86690d66465f262ab244e",
-                "sha256:180511abfef70feb022360b35f4863dd68e08334197089201d5c52208de9ca2e",
-                "sha256:19d52c60dc37520385f538d6d1a4c40bc398e0885f4ed6a36ce10b631dab2852",
-                "sha256:1d559a76ae87143ad96c2ecd6fdd38e691721e175df7ced3fcdc681b4638bca1",
-                "sha256:210ec4a058480b9c3869082e52b66d80c4a48eda9682d7a569a1a5a48100ea54",
-                "sha256:2163d736d6f62b20753be5da3dc07a188420b355f057fcbb3075b05ee6227b2f",
-                "sha256:22ee2c94fee1e391735be63aa1c9af4c69fdcb325ae9e5e4ddff770248ef60a6",
-                "sha256:28633868be21a187702a8613913e13d1987d831529358c29fc6f6670413df040",
-                "sha256:29390c39ca873737689a0749c9c3257aad96b323439b11279fbc0ba8626ec9c5",
-                "sha256:2aeb108da1ed8e066800fb447ba5ae89d560e6773d228398a87825ac3630452d",
-                "sha256:322f6cc7bf23a264151ebc5229a92600c4b55ac83c83c91c9bab1ec92c888a8d",
-                "sha256:34c15f5798f23488e509eae82fbf749c3d17db74379a88c07c869ece1aa806b9",
-                "sha256:3873866534b6527e6863e742eb23ea2a539e3c7ee00ad3f9bec9da27dbaaff6f",
-                "sha256:3dbc67754882d740f17809342892f0b24398770bd99d48c5cb5ba89f5f5dee4e",
-                "sha256:413b18ac2222f5d961eb8d1c8dcca6c6ca176c8613636d8c13aa23abae7f7a21",
-                "sha256:42f9ec9d77358f557fe17cc15e796c4d4d492ede1a30cba3664822cae66e97c5",
-                "sha256:4ac387ac1be71b798d1c372a924f9c30352f30e684e06f086091297352698ac0",
-                "sha256:4ca92e15fcf02e02e7c24b448a16599b98c9d0e6a46cd85cc50804450ebf7245",
-                "sha256:4d959e929cec805c2bf391418b1121590b4e7d5cb00af7b1ba521443d45a0918",
-                "sha256:5091aacbdb667b418b751157f48f6daa17142c4f9063d58e5a64c90b2afbdf9a",
-                "sha256:5a03ae5ac85b04b2034a0689add9ff597b16d5e24066a87f6ab0e9fa67049156",
-                "sha256:5e1341276ce8b7752db9aeac6bbb0cbe82a3f6a6186866bf6b4906d8d328d50b",
-                "sha256:6043d251fac27ca04ff22ed8deb5ff7a43dc18e8a4a15b4c442d2a20fa313162",
-                "sha256:610d5cbbfd026e2f6d15665af51e048e49b68363fedece2ed318cc8fe080dd94",
-                "sha256:622a5157ffcd793d305387c1c9fb94185f496c8c9fd66dafb59de0807bc14ad7",
-                "sha256:65b67637f0a25ac9d25efb13c1578eb065870220ffa82f132c5b2d8e43ac39c3",
-                "sha256:66573c8c7808cce4f3b56c23cb7cad6c3d7f4c464b9016d35f5344ad743896d7",
-                "sha256:66b688fc139c6742057795510e3b12c4acbf90d11af1eff9689a41d9c84478d6",
-                "sha256:685b884fa41bd2913fd20af85866c4ff886b7cbb7e4833b918996aa5d45a04be",
-                "sha256:6a5834e392c97f19f36670e34bf9d346d733ad89ee0689a6419dd737dfa4308a",
-                "sha256:728313cc0d59d1a1a004f675607dcf5c711ced3f55e75d82b3f264fd758869f3",
-                "sha256:733e1cfffc4cd99848230e2999c8a86e284c6af6746482f8ad2ad554dce14e39",
-                "sha256:7814b2cf23aad23464859973c5cd2066ca2fd99e0b934acefbb0b728ac2525bf",
-                "sha256:7c77801620e5e75fb9c7abae235d3cc45d212a67efa98f4972eef63e736a8daa",
-                "sha256:7cd42c66d49ffb68dea065e1c8a4323e7ceab386e660fee9863d4fa227302ba9",
-                "sha256:7d2ae2f7c50adec20fde46a73465de31a6a6fbb4903240f8b7304549752ca7a1",
-                "sha256:7edff02e44dd0badd749d7342e40705a398d98c5d8f7570f57cff9568c2351fa",
-                "sha256:87981008d565f647142869d99915cc4760b7725858da3d39ecb2a606e23f36fd",
-                "sha256:92e2376ce3ca0e3e443b3c5c2bb5d584c7e59221edfb0035313c6306049ba55a",
-                "sha256:950710f7370613a6bfa2ccd842b488c5b8072e83fb6b7d45d99110bf44651d06",
-                "sha256:980527f4ccc6644855bb68056fe7835da6d06d37776a52df5bcc1882df57c3db",
-                "sha256:9fbffc5bad4df99a509783cbd449ed0d24fcd5a450c28e7756c8f20eda3d2aa5",
-                "sha256:a8b02e0119d6ee381a265d8d2450a38096f82916d895fed2dfd81d4c7a54d6e4",
-                "sha256:b17e627844d86031c77147c40bf992a6e1114025a460874deeda6500d0f34862",
-                "sha256:b1aa62903a2c5768b0001632efdea2e8da6c80abdd520c2e8a16001cc9affb23",
-                "sha256:b32e4eed2ef19a20dfb57698497a9bc54e74efb2e260c003e9056c145f130dc7",
-                "sha256:b44fa04720bbfd617b6aef036989c8c30435f11450c0a59136291d7b41ed647f",
-                "sha256:b4535d98df83abebb572035754fb3d4ad09ce7449375fa09fa9ede2dbc87b62b",
-                "sha256:bb6a5777bf558f444cd4883d617546182cfeff8f2d4acd885253f11a16740534",
-                "sha256:bc2eb67387b8376120a2be6cba9d23f9d6a6c3828e00fb0a64c55ad7b54116d1",
-                "sha256:bd351ceb2decd23d523fc50bad631ee9ae6e97e7cdc355ce5600fe310484f96e",
-                "sha256:bf70097bd497089f1baabf9cbb3ec4f69c022dc7a70c41ba9c238fa4d0fff7ab",
-                "sha256:c7fd18d4b7939408df9315fedbdb05e179760960a92b3752498e2fcd03f24c3d",
-                "sha256:cc359e408712faf9ea775f4c0ec8f2bfc843afe47747a657808d9595edd34d71",
-                "sha256:cd8fc35d4c0c717cc29b0cb894871555cb7137a081e179877ecc537e2607f0b9",
-                "sha256:daa44cefde19978af57ac1d50413cd86ebf2b497328e7a27832f5824bda47439",
-                "sha256:db5098587f58fbf8582d9bda2462762b367207246d3e19623782fb449c3c5fcc",
-                "sha256:db6fd53ef5f1914ad801830406440c3bfb701e38a607eda47c38adba267ba300",
-                "sha256:e1414599a97554d451e441afb362dbee1505e4550852c0068370d843757a3fe2",
-                "sha256:ee42a8f850143ae7c67ea09a183a6a4ad8d053e1dbd9a1134e21a7b5c1bc6c73",
-                "sha256:f23abcf6eca5859a2982beadfb5111f8c5e76e30ff99aaee3c1c327f814f9f10",
-                "sha256:f6748c447feeadda059719ef5ab1fb9d84bd370e205b20049a0e8b45ef4ad593"
+                "sha256:03be7ad107d252bb7325d4af6309fdd2c025d08854d35f0e7abc8bf048f4245e",
+                "sha256:071552b065e809d24c5653fcc14968cfd6fde4e279408640d5ac58e3353a3c5f",
+                "sha256:08b8723248730599c9803ae4c97b8f3f76c55219104303c88cb962a31e3bb5ee",
+                "sha256:08bda7b2c522ff9f1e554570da16298271ebb0c56ab9699446aacba249008988",
+                "sha256:0aaf4d44f1f819360f9432df538d54bbf850f18152f34e20337c01b828479171",
+                "sha256:0cabfc297f4cf921f15bc789a8fbfd7115eb9f813d3f47a74b609894bc66ab0d",
+                "sha256:13acf6164ead81c9fc2afa0e1ea6d6134352973ce2bb35496834fee057063c04",
+                "sha256:15b083d1b789b230e5ac284442d9ecb113c93f3785a6824f748befaab803b812",
+                "sha256:161fcd3281c42f644aa8dec7753cca2af03ce654e17d76da4f0dab34a12480ca",
+                "sha256:1a994a42f49dab5b6287e499be7d3d2751776486229980d8857ad53b8333d469",
+                "sha256:20d75ea11527331a2980ab04762a9d960bcfea9475c54bbeab777af880de61cd",
+                "sha256:225c61e08fe517aede7912937939e09adf086c8e6f7e40d4c85ad678c2c2aea3",
+                "sha256:3135dd574ef1286189f3f04a36c8b7a256376914f8cbbce66b94f13125ded858",
+                "sha256:3491c7de09e44eded16824cb58cf9b5cc1dc6f066a0bb7aa69929d02aa53b828",
+                "sha256:3551912f5c34d8dd7c32c6bb00ae04192af47f7b9f653608f107d19c1a21a194",
+                "sha256:38a7b5140a48fc91681cdb5cb95b7cd64640b43d19259fdd707fa9d5a715f2b2",
+                "sha256:3a3498a8326111221560e930f198b495ea6926937e249f475052ffc6893a6680",
+                "sha256:3bfc7689a1bacb9bcd2f2d5185d99507aa29f667a58dd8adaa43b5a348139e46",
+                "sha256:421d13523d11c57f57f257152bc4a6bb463aadf7a3918e9c96fefdd6be8dbfb8",
+                "sha256:424799c71ff435094e5fb823c40eebb4500f0e048133311e9c026467e8ccebac",
+                "sha256:474e21d0e07cd09679e357d1dac76e570dab86665e79a9d3354b10a279ac6fb3",
+                "sha256:4c7e8c8e1e1918dcf6a652ac4b9d87164587c26fd2ce5dd81e73a5ab3b3d492f",
+                "sha256:506a6dab4c7ffdcacdf0b8e70bd20eb2e77fa994519547c9d88d676400fcad58",
+                "sha256:510cd3bfabb63a07405b7b79fae63127e34c118b7531a2cbbafc7a24fd878594",
+                "sha256:517ba47ca04a55b1f50ee8df9fd97f6c37df5537d118fb2718952b8623860466",
+                "sha256:539d4cb1b16b57026999c53e5aab857fe706e70ae5310cc8c232479923f932e6",
+                "sha256:5c36428cc4f7fae56354db7f46677fd21222fc3cb1e8829549b851172033e043",
+                "sha256:5db59223ed1e634d842a053325f85f908359c6dac9c8ddce8ef145061fae7df8",
+                "sha256:5e606846c049ed40940524057bfdf1105af6066688c0e6a1a3ce2038589bae70",
+                "sha256:6060794aac9f7b0644b299f46a9c6cbc0bc470bd01572f4134df140afd41ded6",
+                "sha256:62c29bc36a6d9be68fe7b5aaf1e120b4aa66a958d1e146601fcd583eb12cae7b",
+                "sha256:73326b211e7410c8bd6a74500b1e3f392f39cf10862e243d00937e924f112c01",
+                "sha256:78f07961f4f214ea8e80be63cffd5cc158eb06cd922ffbf6c7155b11728f28f9",
+                "sha256:7c97554ea521f898753d9773891d0347ebfaddcc1dee2ad94850b163171bf1f1",
+                "sha256:8898f6699f740ca93a0879ed07d8e6db02d68af889d0ebb3d13ab017e6b1af1e",
+                "sha256:8a41fdc751dc4707a4fafb111c442411816a7c225ebb5cadb57599534b5d5372",
+                "sha256:8e0004b0393d72d76de94b4792a006cb960c1c65c7659930fbf9a81ce4341982",
+                "sha256:977b1d4f868986b4ba5d03c317fde4d3b66e687d74473130cd598e3103db34fa",
+                "sha256:9a4f6e0b01df820ba9ed0b4e618ca83a1c089e48d4f268d0e00dcd49893d4549",
+                "sha256:9b9298964389c180a063a9e8bac8a80ed42de11d04166b20249bfa0a489e0e0f",
+                "sha256:a08c8b322b671857c81f4c30cd3c8df2895fd3c0e9358714f39e0ef8fb327702",
+                "sha256:ad31f184dcd3271de26ab1f9c51574afb99e1b0e484ab1da3641256b723e4994",
+                "sha256:aff3656af2add93f290731a6b8930b23b35c0c09569150130a58192b3ec6fc61",
+                "sha256:b2f41261b648cf5dee425f37ff14f4ad151c2f24b827052b402637158fd056ef",
+                "sha256:b413117210fa6d92664c3d860571e8e8727c3e8f2ff197276c5d0cb365abd3ad",
+                "sha256:b7efc7e7049ef366777cfd35437c18a4166bb50a5606a1c840ee3b9624b54fc9",
+                "sha256:b8f94acd52e530a38f25e4d5bf7ddfdd4bea9193e718f58419def0d4406b58d3",
+                "sha256:d0a70151d7de8a3194cdc906bcc1a42e14594787c64b0c1c9c975e5a2af3e251",
+                "sha256:d360e5d5dd3d55bf5d1776964625018d85b937d1032bae1926dd52253decd0db",
+                "sha256:d4e62417e89b717a7bcd8576ac3108cd063225942cc91c5b37ff5465fdccd386",
+                "sha256:d65bac5f6724d9ea6f0b5a0f0e4952fbbf209adcf6b5583b54c54bd2fcd74dc0",
+                "sha256:e02beaab433fd1104b2804f909e694cfbdb6578020740a9051597adc1cd4e19f",
+                "sha256:e4b631688dfbdd61b5610e20b64b99d25771c6d52d9da73349342d2a0f11c46a",
+                "sha256:e4e9db78b71db2b1684ee4ecc3e32c4600f18cdf76e6b9ae03e338e52ee4b168",
+                "sha256:eb4d176394c37a76e8b0afe54b12d58614a67a60a7f8c0dd3a5afbb013c01092",
+                "sha256:f08665d3cc5abc2f770f472a9b5f720a9b3ab0b8b3bb97c7c1487515e5653d39",
+                "sha256:f3d851af3852f16ad4adc7ee054fd9c90a7a5063de94d815b7f6a88477b9f4c6",
+                "sha256:f4ba58157e8ae33ee86fadf9062c506e535afd904f07f9be32731f4410a23b7f",
+                "sha256:f664ed7613b8b18f0ce5696b146776266a038c19c5cd6efffa08ecc189b01b73",
+                "sha256:f947b359cc4769af8b49be7e37af01f05fcf15b401da2528021148e4a54426d1",
+                "sha256:fe4189846448df013cd9df11bba38ddf78043f8c290a9f06430732a7a8601cce",
+                "sha256:fea5cb1c63efe1399f0812532c7cf65458d38fd011be350bc5021dfcac39fba8",
+                "sha256:fedf0dee7a412ca6d1d6d92c158fe9cbaa8ea0cae90d268f9ccc0744de7a97d0",
+                "sha256:fffff7bfb6799a763d3742c59c6ee7ffadda21abed557637bc44ed1080876484"
             ],
             "index": "pypi",
-            "version": "==3.11.3"
+            "version": "==3.11.4"
         },
         "pymongo-stubs": {
             "hashes": [
@@ -516,18 +524,18 @@
         },
         "rfc3986": {
             "hashes": [
-                "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d",
-                "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50"
+                "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835",
+                "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"
             ],
-            "version": "==1.4.0"
+            "version": "==1.5.0"
         },
         "six": {
             "hashes": [
-                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
-                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.15.0"
+            "version": "==1.16.0"
         },
         "tqdm": {
             "hashes": [
@@ -582,11 +590,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
-                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
-                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+                "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497",
+                "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
+                "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
             ],
-            "version": "==3.7.4.3"
+            "version": "==3.10.0.0"
         },
         "urllib3": {
             "hashes": [
diff --git a/ietfdata/datatracker.py b/ietfdata/datatracker.py
index 816e6a1..7f08c13 100644
--- a/ietfdata/datatracker.py
+++ b/ietfdata/datatracker.py
@@ -310,7 +310,7 @@ class Submission(Resource):
     auth_key        : str
     authors         : str   # See the parse_authors() method
     checks          : List[SubmissionCheckURI]
-    document_date   : datetime
+    document_date   : Optional[datetime]
     draft           : DocumentURI
     file_size       : Optional[int]
     file_types      : str   # e.g., ".txt,.xml"
@@ -2020,7 +2020,7 @@ def _cache_update(self, obj_type_uri: URI, dt_version_changed:bool) -> None:
 
         meta = self._cache_load_metadata(obj_type_uri)
 
-        # URIs under /api/v1/name/ are internal names used by the datatracker. 
+        # URIs under /api/v1/name/ are internal names used by the datatracker.
         # Delete and recreate these caches if the datatracker version changed.
         if self._hints[obj_type_uri.uri].update_strategy == "V" and dt_version_changed:
             self.log.info(f"_cache_update: drop {obj_type_uri} due to datatracker version change")
diff --git a/ietfdata/datatracker_ext.py b/ietfdata/datatracker_ext.py
index 3634e66..c83f4ac 100644
--- a/ietfdata/datatracker_ext.py
+++ b/ietfdata/datatracker_ext.py
@@ -101,7 +101,7 @@ def __init__(self,
 
 
 
-    def draft_history(self, draft: Document) -> List[DraftHistory]:
+    def draft_history(self, draft: Document, drafts_seen: List[Document] = []) -> List[DraftHistory]:
         """
         Find the previous versions of an Internet-Draft
         """
@@ -109,6 +109,11 @@ def draft_history(self, draft: Document) -> List[DraftHistory]:
 
         drafts : List[DraftHistory] = []
 
+        if draft in drafts_seen:
+            return []
+        else:
+            drafts_seen.append(draft)
+
         # Step 1: Use document_events() to find previous versions of the draft.
         for event in self.document_events(doc=draft, event_type="new_revision"):
             drafts.append(DraftHistory(draft, event.rev, event.time, None))
@@ -144,7 +149,7 @@ def draft_history(self, draft: Document) -> List[DraftHistory]:
                     found = True
                     break
             if not found:
-                drafts.append(DraftHistory(draft, submission.rev, submission.document_date, submission))
+                drafts.append(DraftHistory(draft, submission.rev, submission.submission_date, submission))
 
         # Step 3: Use related_documents() to find additional drafts this replaces:
         for related in self.related_documents(source=draft, relationship_type=self.relationship_type_from_slug("replaces")):
@@ -163,7 +168,7 @@ def draft_history(self, draft: Document) -> List[DraftHistory]:
         # Step 4: Process the drafts this replaces, to find earlier versions:
         for r in replaces:
             if r.name != draft.name:
-                drafts.extend(self.draft_history(r))
+                drafts.extend(self.draft_history(r, drafts_seen=drafts_seen))
 
         return list(reversed(sorted(drafts, key=lambda d: d.date)))
 
diff --git a/ietfdata/mailarchive.py b/ietfdata/mailarchive.py
index d06fc81..e7ecfa8 100644
--- a/ietfdata/mailarchive.py
+++ b/ietfdata/mailarchive.py
@@ -23,22 +23,32 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-import hashlib
-import json
-import re
+from __future__    import annotations
+
 import requests
 import email
-import ietfdata.datatracker as dt
-import abc
 import os
 import logging
+import gridfs
+
 
 from datetime      import datetime, timedelta
-from typing        import List, Optional, Tuple, Dict, Iterator, Type, TypeVar, Any
-from pathlib       import Path
+from typing        import Dict, Iterator, List, Optional, Tuple, Union
+from pymongo       import MongoClient, ASCENDING, ReplaceOne
+from email         import policy, utils
 from email.message import Message
 from imapclient    import IMAPClient
 
+from dataclasses import dataclass, field
+
+import time
+
+from email_reply_parser import EmailReplyParser
+
+from bs4 import BeautifulSoup # type: ignore
+
+import pandas as pd
+
 # =================================================================================================
 # Private helper functions:
 
@@ -51,191 +61,130 @@ def _parse_archive_url(archive_url:str) -> Tuple[str, str]:
 
     return (mailing_list, message_hash)
 
-# =================================================================================================
-# MailArchiveHelper interface:
-
-class MailArchiveHelper(abc.ABC):
-    """
-    Abstract class for mail archive helpers.
-    """
 
-    @property
-    @classmethod
-    @abc.abstractmethod
-    def name(cls):
-        return NotImplementedError
+def _clean_email_text(email : Message) -> str:
+    clean_text_reply = ""
+    try:
+        clean_text_bytes = email.get_payload(decode=True)
+        if email.get_content_charset() is not None:
+            clean_text = clean_text_bytes.decode(email.get_content_charset())
+        else:
+            clean_text = clean_text_bytes.decode("utf-8")
+        clean_text = BeautifulSoup(clean_text, "lxml").text # this fixes some issues even in text/plain mails
+        clean_text_reply = EmailReplyParser.parse_reply(clean_text)
+    except:
+        clean_text_reply = ""
+    return clean_text_reply
+
+
+def _ml_message_from_db_message(list_name: str, imap_uid: int, headers: Dict[str, str], body: str, timestamp: datetime) -> MailingListMessage:
+    return MailingListMessage(list_name,
+                              imap_uid,
+                              headers.get("Message-ID", headers.get("Message-Id", None)),
+                              headers.get("From", headers.get("from", None)),
+                              headers.get("Subject", headers.get("subject", None)),
+                              timestamp,
+                              headers.get("In-Reply-To", None),
+                              headers.get("References", None),
+                              headers.get("Archived-At", None),
+                              headers,
+                              body)
 
-    @property
-    @classmethod
-    @abc.abstractmethod
-    def version(cls):
-        return NotImplementedError
 
+# =================================================================================================
 
-    @property
-    @classmethod
-    @abc.abstractmethod
-    def provided_fields(cls):
-        return NotImplementedError
+@dataclass
+class MailingListMessage:
+    list_name     : Optional[str]
+    _imap_uid     : int
+    message_id    : Optional[str]
+    from_addr     : Optional[str]
+    subject       : Optional[str]
+    date          : datetime
+    in_reply_to   : Optional[str]
+    references    : Optional[str]
+    archived_at   : Optional[str]
+    headers       : Dict[str, str]
+    body          : str
+    parent        : Optional["MailingListMessage"] = None
+    children      : List["MailingListMessage"] = field(default_factory=list)
 
+    def add_child_message(self, child: "MailingListMessage"):
+        self.children.append(child)
 
-    @abc.abstractmethod
-    def scan_message(self, msg: Message) -> Dict[str, Any]:
-        pass
 
+    def get_child_count(self) -> int:
+        return len(self.children) + sum([child.get_child_count() for child in self.children])
 
-    @abc.abstractmethod
-    def filter(self, metadata: Dict[str, Any], **kwargs) -> bool:
-        pass
 
+    def get_child_from_addrs(self, child_from_addrs: List[str]) -> List[str]:
+        if self.from_addr is not None:
+            child_from_addrs.append(self.from_addr)
+        for child in self.children:
+            child.get_child_from_addrs(child_from_addrs)
+        return child_from_addrs
 
-    @abc.abstractmethod
-    def serialise(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        pass
 
+    def get_child_dates(self, child_dates: List[datetime]):
+        child_dates.append(self.date)
+        for child in self.children:
+            child.get_child_dates(child_dates)
+        return child_dates
 
-    @abc.abstractmethod
-    def deserialise(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        pass
 
 # =================================================================================================
 
-class MailingListMessage:
-    message       : Message
-    _metadata     : Dict[str, Any]
-
-    def __init__(self, message: Message, metadata: Dict[str, Any]):
-        self.message = message
-        self._metadata = {}
-        for helper_name in metadata:
-            self._metadata = {**self._metadata, **metadata[helper_name]}
+class MailingListThread:
+    root : MailingListMessage
 
+    def __init__(self, root: MailingListMessage):
+        self.root = root
 
-    def has_metadata(self, name: str) -> bool:
-        return name in self._metadata
 
+    def get_message_count(self):
+        return 1 + self.root.get_child_count()
 
-    def metadata(self, name: str) -> Any:
-        if not self.has_metadata(name):
-            raise Exception(f"Message does not have a metadata field named {name}")
-        return self._metadata.get(name)
 
+    def get_unique_from_count(self):
+        from_addrs = self.root.get_child_from_addrs([])
+        return len(list(set(from_addrs)))
 
-class MessageThread:
-    _msg_ids : List[str]
-    messages: List[Tuple[int, MailingListMessage]]
 
-    def __init__(self, index: int, first_message: MailingListMessage):
-        self._msg_ids =  [first_message.message['Message-ID']]
-        self.messages = [(index, first_message)]
+    def get_duration(self):
+        earliest_date = self.root.date
+        latest_date = sorted(self.root.get_child_dates([]), reverse=True)[0]
+        return latest_date - earliest_date
 
 
-    def should_contain(self, msg: MailingListMessage) -> bool:
-        if "References" in msg.message:
-            for msg_id in msg.message["References"].split():
-                if msg_id in self._msg_ids:
-                    return msg_id in self._msg_ids
-        return msg.message["In-Reply-To"] in self._msg_ids
-
-
-    def append(self, index: int, msg: MailingListMessage) -> None:
-        assert self.should_contain(msg)
-        self._msg_ids.append(msg.message["Message-ID"])
-        self.messages.append((index, msg))
-
 # =================================================================================================
 
 class MailingList:
+    _mail_archive      : MailArchive
     _list_name         : str
-    _cache_dir         : Path
-    _cache_folder      : Path
-    _last_updated      : datetime
     _num_messages      : int
+    _last_updated      : datetime
     _archive_urls      : Dict[str, int]
-    _helpers           : List[MailArchiveHelper]
-    _msg_metadata      : Dict[int, Dict[str, Any]]
-    _cached_metadata   : Dict[str, Dict[str, Any]]
 
-    def __init__(self, cache_dir: Path, list_name: str, helpers: List[MailArchiveHelper] = [], _reuse_imap=None):
+    def __init__(self, mail_archive: MailArchive, list_name: str):
         logging.basicConfig(level=os.environ.get("IETFDATA_LOGLEVEL", "INFO"))
-        self.log            = logging.getLogger("ietfdata")
+        self.log           = logging.getLogger("ietfdata")
+        self._mail_archive = mail_archive
         self._list_name    = list_name
-        self._cache_dir    = cache_dir
-        self._cache_folder = Path(self._cache_dir, "mailing-lists", self._list_name)
-        self._cache_folder.mkdir(parents=True, exist_ok=True)
-        self._num_messages = len(list(self._cache_folder.glob("*.msg")))
+        self._num_messages = self._mail_archive._db.messages.find({"list": self._list_name}).count()
         self._archive_urls = {}
-        self._helpers = helpers
-        self._msg_metadata = {}
-        self._cached_metadata = {}
-
-        # Rebuild the metadata cache:
-        metadata_cache = Path(self._cache_folder, "metadata.json")
-        metadata_cache_tmp = Path(self._cache_folder, "metadata.json.tmp")
-        if metadata_cache.exists():
-            with open(metadata_cache, "r") as metadata_file:
-                self._cached_metadata = json.load(metadata_file)
-                message_metadata = self._cached_metadata["message_metadata"] # type: Dict[str, Dict[str, Dict[str, Any]]]
-                for msg_id_str in message_metadata:
-                    msg_id = int(msg_id_str)
-                    if not Path(self._cache_folder, F"{msg_id:06d}.msg").exists():
-                        self.log.warn(F"dropping metadata for non-existing message {self._list_name}/{msg_id:06d}.msg")
-                        continue
-                    metadata : Dict[str, Dict[str, Any]] = {}
-                    message_text = None
-                    for helper in self._helpers:
-                        if helper.name in self._cached_metadata["helpers"] and helper.version != self._cached_metadata["helpers"][helper.name]:
-                            self.log.info(F"{helper.name}: version changed, discarding cached metadata")
-                            (message_metadata[msg_id_str]).pop(helper.name)
-                        if helper.name not in message_metadata[msg_id_str] or not all(metadata_field in message_metadata[msg_id_str][helper.name] for metadata_field in helper.provided_fields):
-                            if message_text is None:
-                                message_text = self.raw_message(msg_id)
-                            self.log.info(F"scan message {self._list_name}/{msg_id:06} for metadata - {helper.name:15} (cache update)")
-                            metadata[helper.name] = helper.scan_message(message_text)
-                        else:
-                            metadata[helper.name] = helper.deserialise(message_metadata[msg_id_str][helper.name])
-                    self._msg_metadata[msg_id] = metadata
-        else:
-            self.log.info(F"no metadata cache for mailing list {self._list_name}")
-            for index in self.message_indices():
-                self._msg_metadata[index] = {}
-
-        last_keepalive = datetime.now()
-        for msg_id in self.message_indices():
-            curr_keepalive = datetime.now()
-            if (curr_keepalive - last_keepalive) > timedelta(seconds=10):
-                if _reuse_imap is not None:
-                    self.log.info("imap keepalive")
-                    _reuse_imap.noop()
-                    last_keepalive = curr_keepalive
-            if msg_id not in self._msg_metadata:
-                self._msg_metadata[msg_id] = {}
-            message_text = None
-            for helper in self._helpers:
-                if helper.name not in self._msg_metadata[msg_id] or not all(metadata_field in self._msg_metadata[msg_id][helper.name] for metadata_field in helper.provided_fields):
-                    if message_text is None:
-                        message_text = self.raw_message(msg_id)
-                    self.log.info(F"scan message {self._list_name}/{msg_id:06} for metadata - {helper.name:15} (cache create)")
-                    self._msg_metadata[msg_id][helper.name] = helper.scan_message(message_text)
-        self.serialise_metadata()
 
         # Rebuild the archived-at cache:
-        aa_cache     = Path(self._cache_folder, "aa-cache.json")
-        aa_cache_tmp = Path(self._cache_folder, "aa-cache.json.tmp")
-        if aa_cache.exists():
-            with open(aa_cache, "r") as cache_file:
-                self._archive_urls = json.load(cache_file)
+        aa_cache = self._mail_archive._db.aa_cache.find_one({"list": self._list_name})
+        if aa_cache:
+            self._archive_urls = aa_cache["archive_urls"]
         else:
             self.log.info(F"no archived-at cache for mailing list {self._list_name}")
-            for index, msg in self.messages():
-                if msg.message["Archived-At"] is not None:
-                    self.log.info(F"scan message {self._list_name}/{index:06} for archived-at")
-                    list_name, msg_hash = _parse_archive_url(msg.message["Archived-At"])
-                    self._archive_urls[msg_hash] = index
-            with open(aa_cache_tmp, "w") as cache_file:
-                json.dump(self._archive_urls, cache_file, indent=4)
-            aa_cache_tmp.rename(aa_cache)
-
+            for msg in self.messages():
+                if msg.archived_at is not None:
+                    self.log.info(F"scan message {self._list_name}/{msg._imap_uid} for archived-at")
+                    list_name, msg_hash = _parse_archive_url(msg.archived_at)
+                    self._archive_urls[msg_hash] = msg._imap_uid
+            self._mail_archive._db.aa_cache.replace_one({"list" : self._list_name}, {"list" : self._list_name, "archive_urls": self._archive_urls}, upsert=True)
 
 
     def name(self) -> str:
@@ -246,133 +195,124 @@ def num_messages(self) -> int:
         return self._num_messages
 
 
-    def serialise_metadata(self) -> None:
-        metadata_cache = Path(self._cache_folder, "metadata.json")
-        metadata_cache_tmp = Path(self._cache_folder, "metadata.json.tmp")
-        with open(metadata_cache_tmp, "w") as metadata_file:
-            serialised_metadata = {"helpers": {}, "message_metadata": {}} # type: Dict[str, Dict[Any, Any]]
-            serialised_metadata["helpers"] = {}
-            serialised_metadata["helpers"] = {**serialised_metadata["helpers"], **self._cached_metadata.get("helpers", {})}
-            for helper in self._helpers:
-                serialised_metadata["helpers"][helper.name] = helper.version
-            for msg_id in self._msg_metadata:
-                serialised_metadata["message_metadata"][msg_id] = self.serialise_message(msg_id)
-                serialised_metadata["message_metadata"][msg_id] = {**serialised_metadata["message_metadata"][msg_id], **(self._cached_metadata.get("message_metadata", {}).get(str(msg_id), {}))}
-            json.dump(serialised_metadata, metadata_file, indent=4)
-        metadata_cache_tmp.rename(metadata_cache)
-
-
-    def serialise_message(self, msg_id: int) -> Dict[str, Dict[str, Any]]:
-        metadata : Dict[str, Dict[str, Any]] = {}
-        for helper in self._helpers:
-            metadata[helper.name] = helper.serialise(self._msg_metadata[msg_id][helper.name])
-        return metadata
-
-
     def raw_message(self, msg_id: int) -> Message:
-        cache_file = Path(self._cache_folder, "{:06d}.msg".format(msg_id))
-        with open(cache_file, "rb") as inf:
-            message = email.message_from_binary_file(inf)
+        cache_metadata = self._mail_archive._db.messages.find_one({"list" : self._list_name, "imap_uid": msg_id})
+        if cache_metadata:
+            message = email.message_from_bytes(self._mail_archive._fs.get(cache_metadata["gridfs_id"]).read(), policy=policy.default)
         return message
 
 
     def message_indices(self) -> List[int]:
-        return [int(str(msg_path).split("/")[-1][:-4]) for msg_path in sorted(self._cache_folder.glob("*.msg"))]
+        cache_metadata = self._mail_archive._db.messages.find({"list" : self._list_name})
+        indices = [message_metadata["imap_uid"] for message_metadata in cache_metadata]
+        return sorted(indices)
 
 
-    def message_from_archive_url(self, archive_url:str) -> MailingListMessage:
+    def message_from_archive_url(self, archive_url:str) -> Optional[MailingListMessage]:
         list_name, msg_hash = _parse_archive_url(archive_url)
         assert list_name == self._list_name
         return self.message(self._archive_urls[msg_hash])
 
 
-    def message(self, msg_id: int) -> MailingListMessage:
-        return MailingListMessage(self.raw_message(msg_id), self._msg_metadata[msg_id])
-
-
-    def messages(self, **kwargs) -> Iterator[Tuple[int, MailingListMessage]]:
-        for msg_id in self.message_indices():
-            include_msg = True
-            for helper in self._helpers:
-                if not helper.filter(self._msg_metadata[msg_id][helper.name], **kwargs):
-                    include_msg = False
-                    break
-            if include_msg:
-                yield msg_id, self.message(msg_id)
+    def message(self, msg_id: int) -> Optional[MailingListMessage]:
+        message = self._mail_archive._db.messages.find_one({"list" : self._list_name, "imap_uid": msg_id})
+        if message is not None:
+            return _ml_message_from_db_message(message["list"], message["imap_uid"], message["headers"], message["body"], message["timestamp"])
+        else:
+            return None
 
 
-    def threads(self) -> List[MessageThread]:
-        threads : List[MessageThread] = []
-        for index, message in self.messages():
-            threaded = False
-            for thread in threads:
-                if thread.should_contain(message):
-                    thread.append(index, message)
-                    threaded = True
-                if threaded:
-                    break
-            if not threaded:
-                threads.append(MessageThread(index, message))
-        return threads
+    def messages(self,
+                 since : str = "1970-01-01T00:00:00",
+                 until : str = "2038-01-19T03:14:07") -> Iterator[MailingListMessage]:
+        messages = self._mail_archive._db.messages.find({"list"     : self._list_name,
+                                           "timestamp": {"$gt": datetime.strptime(since, "%Y-%m-%dT%H:%M:%S"),
+                                                         "$lt": datetime.strptime(until, "%Y-%m-%dT%H:%M:%S")}
+                                          },
+                                          no_cursor_timeout=True)
+        for message in messages:
+            yield _ml_message_from_db_message(message["list"], message["imap_uid"], message["headers"], message["body"], message["timestamp"])
+        messages.close()
 
 
-    def update(self, _reuse_imap=None) -> List[int]:
+    def update(self, reuse_imap=None) -> List[int]:
         new_msgs = []
-        if _reuse_imap is None:
+        last_keepalive = datetime.now()
+        if reuse_imap is None:
             imap = IMAPClient(host='imap.ietf.org', ssl=False, use_uid=True)
             imap.login("anonymous", "anonymous")
         else:
-            imap = _reuse_imap
+            imap = reuse_imap
         imap.select_folder("Shared Folders/" + self._list_name, readonly=True)
 
         msg_list  = imap.search()
         msg_fetch = []
 
+        cached_messages = {msg["imap_uid"] : msg for msg in self._mail_archive._db.messages.find({"list": self._list_name})}
+
+        cache_replaces : List[Union[ReplaceOne]] = []
+
         for msg_id, msg in imap.fetch(msg_list, "RFC822.SIZE").items():
-            cache_file = Path(self._cache_folder, F"{msg_id:06d}.msg")
-            if not cache_file.exists():
+            curr_keepalive = datetime.now()
+            if msg_id not in cached_messages:
+                msg_fetch.append(msg_id)
+            elif cached_messages[msg_id]["size"] != msg[b"RFC822.SIZE"]:
+                self.log.warn(F"message size mismatch: {self._list_name}/{msg_id:06d}.msg ({cached_messages[msg_id]['size']} != {msg[b'RFC822.SIZE']})")
+                cache_file = self._mail_archive._fs.get(cached_messages[msg_id]["gridfs_id"])
+                cache_file.delete()
+                self._mail_archive._db.messages.delete_one({"list" : self._list_name, "imap_uid" : msg_id})
                 msg_fetch.append(msg_id)
-            else:
-                file_size = cache_file.stat().st_size
-                imap_size = msg[b"RFC822.SIZE"]
-                if file_size != imap_size:
-                    self.log.warn(F"message size mismatch: {self._list_name}/{msg_id:06d}.msg ({file_size} != {imap_size})")
-                    cache_file.unlink()
-                    msg_fetch.append(msg_id)
 
         if len(msg_fetch) > 0:
-            aa_cache     = Path(self._cache_folder, "aa-cache.json")
-            aa_cache_tmp = Path(self._cache_folder, "aa-cache.json.tmp")
-            aa_cache.unlink()
-
             for msg_id, msg in imap.fetch(msg_fetch, "RFC822").items():
-                cache_file = Path(self._cache_folder, F"{msg_id:06d}.msg")
-                fetch_file = Path(self._cache_folder, F"{msg_id:06d}.msg.download")
-                if not cache_file.exists():
-                    with open(fetch_file, "wb") as outf:
-                        outf.write(msg[b"RFC822"])
-                    fetch_file.rename(cache_file)
-
-                    e = email.message_from_bytes(msg[b"RFC822"])
-                    if e["Archived-At"] is not None:
-                        list_name, msg_hash = _parse_archive_url(e["Archived-At"])
-                        self._archive_urls[msg_hash] = msg_id
-                    self._num_messages += 1
-                    new_msgs.append(msg_id)
-
-                    self._msg_metadata[msg_id] = {}
-                    for helper in self._helpers:
-                        self.log.info(F"{helper.name}: scan message {self._list_name}/{msg_id:06} for metadata")
-                        self._msg_metadata[msg_id][helper.name] = helper.scan_message(e)
-
-            with open(aa_cache_tmp, "w") as aa_cache_file:
-                json.dump(self._archive_urls, aa_cache_file)
-            aa_cache_tmp.rename(aa_cache)
-
-        self.serialise_metadata()
+                cache_file_id = self._mail_archive._fs.put(msg[b"RFC822"])
+                e = email.message_from_bytes(msg[b"RFC822"], policy=policy.default)
+                if e["Archived-At"] is not None:
+                    list_name, msg_hash = _parse_archive_url(e["Archived-At"])
+                    self._archive_urls[msg_hash] = msg_id
+                self._num_messages += 1
+                timestamp = None # type: Optional[datetime]
+                try:
+                    msg_date = email.utils.parsedate(e["Date"]) # type: Optional[Tuple[int, int, int, int, int, int, int, int, int]]
+                    if msg_date is not None:
+                        timestamp = datetime.fromtimestamp(time.mktime(msg_date))
+                    else:
+                        timestamp = None
+                except:
+                    timestamp = None
+                try:
+                    headers = {name : value for name, value in e.items()}
+                except:
+                    headers = {}
+                cache_replaces.append(ReplaceOne({"list" : self._list_name, "id": msg_id},
+                                                 {"list"       : self._list_name,
+                                                  "imap_uid"   : msg_id,
+                                                  "gridfs_id"  : cache_file_id,
+                                                  "size"       : len(msg[b"RFC822"]),
+                                                  "timestamp"  : timestamp,
+                                                  "headers"    : headers,
+                                                  "body"       : _clean_email_text(e)},
+                                                 upsert=True))
+
+                if len(cache_replaces) > 1000:
+                    self._mail_archive._db.messages.bulk_write(list(cache_replaces))
+                    cache_replaces = []
+
+                curr_keepalive = datetime.now()
+                if (curr_keepalive - last_keepalive) > timedelta(seconds=10):
+                    self.log.info("imap keepalive")
+                    imap.noop()
+                    last_keepalive = curr_keepalive
+
+                new_msgs.append(msg_id)
+
+            self._mail_archive._db.aa_cache.replace_one({"list" : self._list_name}, {"list" : self._list_name, "archive_urls": self._archive_urls}, upsert=True)
+
+        if len(cache_replaces) > 0:
+            result = self._mail_archive._db.messages.bulk_write(list(cache_replaces))
 
         imap.unselect_folder()
-        if _reuse_imap is None:
+        if reuse_imap is None:
             imap.logout()
         self._last_updated = datetime.now()
         return new_msgs
@@ -382,20 +322,73 @@ def last_updated(self) -> datetime:
         return self._last_updated
 
 
+    def messages_dataframe(self,
+                 since : str = "1970-01-01T00:00:00",
+                 until : str = "2038-01-19T03:14:07") -> pd.DataFrame:
+        messages = self.messages(since = since, until = until)
+        messages_as_dict = []
+        for message in messages:
+            messages_as_dict.append({"Message-ID": message.message_id,
+                                     "From" : message.from_addr,
+                                     "Subject": message.subject,
+                                     "Date": message.date,
+                                     "In-Reply-To": message.in_reply_to,
+                                     "References": message.references,
+                                     "Body": message.body})
+        df = pd.DataFrame(data=messages_as_dict)
+        df.set_index("Message-ID", inplace=True)
+        return df
+
+
+    def threads(self) -> Iterator[MailingListThread]:
+        threads = []
+        message_by_message_id = {message.message_id : message for message in self.messages()}
+
+        for message_id in message_by_message_id:
+            message = message_by_message_id[message_id]
+            if message.in_reply_to is None:
+                threads.append(message)
+            if message.in_reply_to is not None and message.in_reply_to in message_by_message_id:
+                message.parent = message_by_message_id[message.in_reply_to]
+                message_by_message_id[message.in_reply_to].add_child_message(message)
+
+        return iter([MailingListThread(message) for message in threads])
+
+
 # =================================================================================================
 
 class MailArchive:
-    _cache_dir     : Path
-    _mailing_lists : Dict[str,MailingList]
-    _helpers       : List[MailArchiveHelper]
+    _mailing_lists : Dict[str, MailingList]
 
-    def __init__(self, cache_dir: Path, helpers: List[MailArchiveHelper] = []):
+
+    def __init__(self, mongodb_hostname: str = "localhost", mongodb_port: int = 27017, mongodb_username: Optional[str] = None, mongodb_password: Optional[str] = None):
         logging.basicConfig(level=os.environ.get("IETFDATA_LOGLEVEL", "INFO"))
         self.log            = logging.getLogger("ietfdata")
-        self._cache_dir     = cache_dir
         self._mailing_lists = {}
-        self._helpers       = helpers
+        self._last_full_update = None
+        self._cache_version = "1.0"
+
+        cache_host = os.environ.get('IETFDATA_CACHE_HOST')
+        cache_port = os.environ.get('IETFDATA_CACHE_PORT', 27017)
+        cache_username = os.environ.get('IETFDATA_CACHE_USER')
+        cache_password = os.environ.get('IETFDATA_CACHE_PASSWORD')
+        if cache_host is not None:
+            mongodb_hostname = cache_host
+        if cache_port is not None:
+            mongodb_port = int(cache_port)
+        if cache_username is not None:
+            mongodb_username = cache_username
+        if cache_password is not None:
+            mongodb_password = cache_password
+
+        if mongodb_username is not None:
+            self._db = MongoClient(host=mongodb_hostname, port=mongodb_port, username=mongodb_username, password=mongodb_password).ietfdata_mailarchive
+        else:
+            self._db = MongoClient(host=mongodb_hostname, port=mongodb_port).ietfdata_mailarchive
 
+        self._fs = gridfs.GridFS(self._db)
+
+        self._check_cache_version()
 
     def mailing_list_names(self) -> Iterator[str]:
         imap = IMAPClient(host='imap.ietf.org', ssl=True, use_uid=True)
@@ -407,13 +400,25 @@ def mailing_list_names(self) -> Iterator[str]:
         imap.logout()
 
 
-    def mailing_list(self, mailing_list_name: str, _reuse_imap=None) -> MailingList:
+    def _check_cache_version(self):
+        # check if cache pre-dates versioning; if so, set to 1.0
+        if "cache_info" not in self._db.list_collection_names():
+            self.log.info("Setting cache version to 1.0")
+            self._db.cache_info.insert_one({"list": "__cache_version__", "version" : "1.0", "last_imap_update": None})
+            self._db.messages.create_index([('list', ASCENDING), ('imap_uid', ASCENDING)], unique=True)
+            self._db.messages.create_index([('list', ASCENDING)], unique=False)
+            self._db.messages.create_index([('timestamp', ASCENDING)], unique=False)
+            self._db.aa_cache.create_index([('list', ASCENDING)], unique=True)
+            self._db.metadata_cache.drop()
+
+
+    def mailing_list(self, mailing_list_name: str) -> MailingList:
         if not mailing_list_name in self._mailing_lists:
-            self._mailing_lists[mailing_list_name] = MailingList(self._cache_dir, mailing_list_name, self._helpers, _reuse_imap)
+            self._mailing_lists[mailing_list_name] = MailingList(self, mailing_list_name)
         return self._mailing_lists[mailing_list_name]
 
 
-    def message_from_archive_url(self, archive_url: str) -> MailingListMessage:
+    def message_from_archive_url(self, archive_url: str) -> Optional[MailingListMessage]:
         if "//www.ietf.org/mail-archive/web/" in archive_url:
             # This is a legacy mail archive URL. If we retrieve it, the
             # server should redirect us to the current archive location.
@@ -444,16 +449,20 @@ def download_all_messages(self) -> None:
         imap.login("anonymous", "anonymous")
         for index, ml_name in enumerate(ml_names):
             print(F"Updating list {index+1:4d}/{num_list:4d}: {ml_name} ", end="", flush=True)
-            ml = self.mailing_list(ml_name, _reuse_imap=imap)
-            nm = ml.update(_reuse_imap=imap)
+            ml = self.mailing_list(ml_name)
+            nm = ml.update(reuse_imap=imap)
             print(F"({ml.num_messages()} messages; {len(nm)} new)")
         imap.logout()
 
 
-    def messages(self, **kwargs) -> Iterator[Tuple[Tuple[str, int], MailingListMessage]]:
-        for mailing_list in self._mailing_lists:
-            for msg_id, msg in self._mailing_lists[mailing_list].messages(**kwargs):
-                yield ((mailing_list, msg_id), msg)
+    def messages(self,
+                 since : str = "1970-01-01T00:00:00",
+                 until : str = "2038-01-19T03:14:07") -> Iterator[MailingListMessage]:
+        messages = self._db.messages.find({"timestamp": {"$gt": datetime.strptime(since, "%Y-%m-%dT%H:%M:%S"),
+                                                         "$lt": datetime.strptime(until, "%Y-%m-%dT%H:%M:%S")}
+                                          })
+        for message in messages:
+            yield _ml_message_from_db_message(message["list"], message["imap_uid"], message["headers"], message["body"], message["timestamp"])
 
 
 # =================================================================================================
diff --git a/ietfdata/mailhelper_datatracker.py b/ietfdata/mailhelper_datatracker.py
deleted file mode 100644
index d64b2ad..0000000
--- a/ietfdata/mailhelper_datatracker.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# Copyright (C) 2020 University of Glasgow
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-#    this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-import logging
-import ietfdata.datatracker as dt
-
-from ietfdata.mailarchive import *
-
-class DatatrackerMailHelper(MailArchiveHelper):
-    name    = "Datatracker"
-    version = "r1"
-    provided_fields = ["from_person", "related_docs"]
-
-    def __init__(self, datatracker: dt.DataTracker):
-        logging.basicConfig(level=os.environ.get("IETFDATA_LOGLEVEL", "INFO"))
-        self.log = logging.getLogger("ietfdata")
-        self.dt = datatracker
-
-
-    def scan_message(self, msg: Message) -> Dict[str, Any]:
-        from_name, from_addr = email.utils.parseaddr(str(msg["From"]).replace("\uFFFD", "?"))
-        try:
-            from_name = str(email.header.make_header(email.header.decode_header(from_name)))
-        except:
-            pass
-        from_person = self.dt.person_from_email(from_addr)
-        docs = []
-
-        try:
-            msg_str = msg.as_string()
-        except:
-            self.log.error(f"DatatrackerMailHelper: could not parse message as string")
-            msg_str = ""
-
-        # FIXME: Does this find messages with a document in the subject line?
-        # FIXME: It would be interesting to also find people mentioned in messages
-        draft_matches = re.findall(r'draft-(?P<name>[a-zA-Z0-9_\-]+)-(?P<revision>[0-9_\-]+)', msg_str)
-        for draft_match in draft_matches:
-            doc = self.dt.document_from_draft(f"draft-{draft_match[0]}")
-            if doc is not None and doc not in docs:
-                docs.append(doc)
-        rfc_matches = re.findall(r'(rfc|RFC)(\s)?(?P<number>[0-9]+)', msg_str)
-        for rfc_match in rfc_matches:
-            doc = self.dt.document_from_rfc(f"rfc{rfc_match[-1]}")
-            if doc is not None and doc not in docs:
-                docs.append(doc)
-        return {"from_person": from_person, "related_docs": docs}
-
-
-    def filter(self,
-               metadata    : Dict[str, Any],
-               from_person : Optional[dt.Person] = None,
-               related_doc : Optional[dt.Document] = None,
-               **kwargs) -> bool:
-        return (from_person is None or metadata["from_person"] == from_person) and (related_doc is None or related_doc in metadata["related_docs"])
-
-
-    def serialise(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        return {"from_person"  : metadata["from_person"].resource_uri.uri if metadata["from_person"] is not None else "",
-                "related_docs" : [str(doc.resource_uri.uri) for doc in metadata["related_docs"]]}
-
-
-    def deserialise(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        from_person = self.dt.person(dt.PersonURI(metadata["from_person"])) if metadata["from_person"] != "" else None
-        related_docs = [self.dt.document(dt.DocumentURI(doc_uri)) for doc_uri in metadata["related_docs"]]
-        return {"from_person" : from_person, "related_docs" : related_docs}
diff --git a/ietfdata/mailhelper_headerdata.py b/ietfdata/mailhelper_headerdata.py
deleted file mode 100644
index eed444e..0000000
--- a/ietfdata/mailhelper_headerdata.py
+++ /dev/null
@@ -1,102 +0,0 @@
-# Copyright (C) 2020 University of Glasgow
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-#    this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-import email.header
-import email.utils
-import time
-import logging
-
-from ietfdata.mailarchive        import *
-
-class HeaderDataMailHelper(MailArchiveHelper):
-    name    = "HeaderData"
-    version = "r2"
-    provided_fields = ["from_name", "from_addr", "timestamp", "subject", "message_id", "in_reply_to", "references"]
-
-    def __init__(self):
-        logging.basicConfig(level=os.environ.get("IETFDATA_LOGLEVEL", "INFO"))
-        self.log = logging.getLogger("ietfdata")
-
-
-    def scan_message(self, msg: Message) -> Dict[str, Any]:
-        from_name, from_addr = email.utils.parseaddr(str(msg["From"]).replace("\uFFFD", "?"))
-        try:
-            from_name = str(email.header.make_header(email.header.decode_header(from_name)))
-        except:
-            pass
-        timestamp = None
-        try:
-            msg_date = email.utils.parsedate(msg["Date"]) # type: Optional[Tuple[int, int, int, int, int, int, int, int, int]]
-            if msg_date is not None:
-                timestamp = datetime.fromtimestamp(time.mktime(msg_date))
-        except Exception as e:
-            self.log.error(f"HeaderDataMailHelper: could not parse message `Date` header - {msg['Date']}")
-            pass
-        return {"from_name"   : from_name,
-                "from_addr"   : from_addr,
-                "timestamp"   : timestamp,
-                "subject"     : str(msg["Subject"]),
-                "message_id"  : str(msg["Message-ID"]),
-                "in_reply_to" : str(msg["In-Reply-To"]),
-                "references"  : str(msg["References"])}
-
-
-    def filter(self,
-               metadata   : Dict[str, Any],
-               from_name  : Optional[str] = None,
-               from_addr  : Optional[str] = None,
-               timestamp  : Optional[str] = None,
-               subject    : Optional[str] = None,
-               message_id : Optional[str] = None,
-               in_reply_to: Optional[str] = None,
-               references : Optional[str] = None,
-               **kwargs) -> bool:
-        return ((from_name   is None or metadata["from_name"]   == from_name)   and
-                (from_addr   is None or metadata["from_addr"]   == from_addr)   and
-                (timestamp   is None or metadata["timestamp"]   == timestamp)   and
-                (subject     is None or metadata["subject"]     == subject)     and
-                (message_id  is None or metadata["message_id"]  == message_id)  and
-                (in_reply_to is None or metadata["in_reply_to"] == in_reply_to) and
-                (references  is None or metadata["references"]  == references))
-
-
-    def serialise(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        return {"from_name"   : metadata["from_name"],
-                "from_addr"   : metadata["from_addr"],
-                "timestamp"   : metadata["timestamp"].isoformat() if metadata["timestamp"] is not None else None,
-                "subject"     : metadata["subject"],
-                "message_id"  : metadata["message_id"],
-                "in_reply_to" : metadata["in_reply_to"],
-                "references"  : metadata["references"]}
-
-
-    def deserialise(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
-        return {"from_name"   : metadata["from_name"],
-                "from_addr"   : metadata["from_addr"],
-                "timestamp"   : datetime.fromisoformat(metadata["timestamp"]) if metadata["timestamp"] is not None else None,
-                "subject"     : metadata["subject"],
-                "message_id"  : metadata["message_id"],
-                "in_reply_to" : metadata["in_reply_to"],
-                "references"  : metadata["references"]}
diff --git a/stubs/email_reply_parser/__init__.pyi b/stubs/email_reply_parser/__init__.pyi
new file mode 100644
index 0000000..0be9ef7
--- /dev/null
+++ b/stubs/email_reply_parser/__init__.pyi
@@ -0,0 +1,36 @@
+from typing import Any
+
+class EmailReplyParser:
+    @staticmethod
+    def read(text: Any): ...
+    @staticmethod
+    def parse_reply(text: Any): ...
+
+class EmailMessage:
+    SIG_REGEX: Any = ...
+    QUOTE_HDR_REGEX: Any = ...
+    QUOTED_REGEX: Any = ...
+    HEADER_REGEX: Any = ...
+    MULTI_QUOTE_HDR_REGEX: Any = ...
+    MULTI_QUOTE_HDR_REGEX_MULTILINE: Any = ...
+    fragments: Any = ...
+    fragment: Any = ...
+    text: Any = ...
+    found_visible: bool = ...
+    def __init__(self, text: Any) -> None: ...
+    lines: Any = ...
+    def read(self): ...
+    @property
+    def reply(self): ...
+    def quote_header(self, line: Any): ...
+
+class Fragment:
+    signature: bool = ...
+    headers: Any = ...
+    hidden: bool = ...
+    quoted: Any = ...
+    lines: Any = ...
+    def __init__(self, quoted: Any, first_line: Any, headers: bool = ...) -> None: ...
+    def finish(self) -> None: ...
+    @property
+    def content(self): ...


Test patch:
diff --git a/tests/test_datatracker.py b/tests/test_datatracker.py
index 0ecff3f..05e73cb 100644
--- a/tests/test_datatracker.py
+++ b/tests/test_datatracker.py
@@ -1660,7 +1660,7 @@ def test_group_role_histories(self) -> None:
 
     def test_group_role_histories_email(self) -> None:
         group_role_histories = list(self.dt.group_role_histories(email="csp@csperkins.org"))
-        self.assertEqual(len(group_role_histories), 42)
+        self.assertEqual(len(group_role_histories), 44)
 
 
     def test_group_role_histories_group(self) -> None:
@@ -1679,7 +1679,7 @@ def test_group_role_histories_name(self) -> None:
 
     def test_group_role_histories_person(self) -> None:
         group_role_histories = list(self.dt.group_role_histories(person=self.dt.person(PersonURI("/api/v1/person/person/20209/"))))
-        self.assertEqual(len(group_role_histories), 42)
+        self.assertEqual(len(group_role_histories), 44)
 
 
     def test_group_state_change_event(self) -> None: