Repository: datactive/bigbang (language: python)

Issue:
Integrate listserv 16.5 ingress into standard collect_mail.py



Discussion / hints:

Fix patch:
diff --git a/bigbang/listserv.py b/bigbang/listserv.py
index fdc75099..ae9a0170 100644
--- a/bigbang/listserv.py
+++ b/bigbang/listserv.py
@@ -95,7 +95,7 @@ class ListservMessage:
         "fromaddr": None,
         "toname": None,
         "toaddr": None,
-        "date": None,
+        "date": "Mon, 11 Jan 1111 11:11:11",
         "contenttype": None,
     }
 
@@ -219,7 +219,7 @@ def get_header_from_listserv_file(
                     if not re.match(r"\S+:\s+\S+", content[lnr + 1]):
                         value += " " + content[lnr + 1].strip().rstrip("\n")
                 header[key.lower()] = value
-
+        
         header = cls.format_header_content(header)
         header = cls.remove_unwanted_header_content(header)
         return header
@@ -292,12 +292,33 @@ def get_body_from_html(
 
     @classmethod
     def format_header_content(cls, header: Dict[str, str]) -> Dict[str, str]:
-        header["fromname"] = cls.get_name(header["from"]).strip()
-        header["fromaddr"] = cls.get_addr(header["from"])
-        header["toname"] = cls.get_name(header["reply-to"]).strip()
-        header["toaddr"] = cls.get_addr(header["reply-to"])
-        header["date"] = cls.get_date(header["date"])
-        header["contenttype"] = header["content-type"]
+        "Formats LISTSERV 16.5 header fields to mbox convention."
+        # define formatting configuration
+        formatting_header_conf = {
+            "from": {
+                "fromname": cls.get_name,
+                "fromaddr": cls.get_addr,
+            },
+            "reply-to": {
+                "toname": cls.get_name,
+                "toaddr": cls.get_addr,
+            },
+            "date": {"date": cls.get_date},
+            "content-type": {"contenttype": None},
+        }
+        # run through format settings
+        for key_old, value in formatting_header_conf.items():
+            # if header contains field
+            if key_old in header.keys():
+                for key_new, fct in formatting_header_conf[key_old].items():
+                    if fct is None or header[key_old] is None:
+                        header[key_new] = header[key_old]
+                    else:
+                        header[key_new] = fct(header[key_old])
+            # if header misses field
+            else:
+                for key_new in formatting_header_conf[key_old].keys():
+                    header[key_new] = cls.empty_header[key_new]
         return header
 
     @classmethod
@@ -320,17 +341,17 @@ def get_name(line: str) -> str:
             name = re.sub(r"[^a-zA-Z0-9]+", " ", name)
         else:
             name = line
-        return name
+        return name.strip()
 
     @staticmethod
-    def get_addr(line: str) -> str:
+    def get_addr(line: str) -> Union[str, None]:
         # get string in between < and >
-        email_of_sender = re.findall(r"\<(.*)\>", line)
-        if email_of_sender:
-            email_of_sender = email_of_sender[0]
+        email_addr = re.findall(r"\<(.*)\>", line)
+        if email_addr:
+            email_addr = email_addr[0].strip()
         else:
-            email_of_sender = None
-        return email_of_sender
+            email_addr = ListservMessage.empty_header["fromaddr"]
+        return email_addr
 
     @staticmethod
     def get_date(line: str) -> str:
@@ -340,12 +361,14 @@ def get_date(line: str) -> str:
             line, "%a, %d %b %Y %H:%M:%S"
         )
         return date_time_obj.strftime("%c")
-
+    
     @staticmethod
     def create_message_id(
-        date: str,
-        from_address: str,
+        date: Union[str, None],
+        from_address: Union[str, None],
     ) -> str:
+        #if date is None:
+        #    date = 'Wed Jan 11 11:11:11 1111'
         message_id = (".").join([date, from_address])
         # remove special characters
         message_id = re.sub(r"[^a-zA-Z0-9]+", "", message_id)
@@ -815,6 +838,7 @@ def to_mbox(self, dir_out: str, filename: Optional[str] = None):
             filepath = f"{dir_out}/{self.name}.mbox"
         else:
             filepath = f"{dir_out}/{filename}.mbox"
+        logger.info(f"The list {self.name} is save at {filepath}.")
         first = True
         for msg in self.messages:
             if first:
@@ -889,7 +913,8 @@ def from_url(
         url_login: str = "https://list.etsi.org/scripts/wa.exe?LOGON",
         login: Optional[Dict[str, str]] = {"username": None, "password": None},
         session: Optional[str] = None,
-        instant_dump: bool = True,
+        instant_save: bool = True,
+        only_mlist_urls: bool = True,
     ) -> "ListservArchive":
         """
         Create ListservArchive from a given URL.
@@ -898,7 +923,18 @@ def from_url(
             name:
             url_root:
             url_home:
-            select:
+            select: Selection criteria that can filter messages by:
+                - content, i.e. header and/or body
+                - period, i.e. written in a certain year, month, week-of-month
+            url_login: URL to the login page
+            login: login keys {"username": str, "password": str}
+            session: if auth-session was already created externally
+            instant_save: Boolean giving the choice to save a `ListservList` as
+                soon as it is completely scraped or collect entire archive. The
+                prior is recommended if a large number of mailing lists are
+                scraped which can require a lot of memory and time.
+            only_list_urls: Boolean giving the choice to collect only `ListservList`
+                URLs or also their contents.
         """
         session = get_auth_session(url_login, **login)
         lists = cls.get_lists_from_url(
@@ -906,9 +942,17 @@ def from_url(
             url_home,
             select,
             session,
-            instant_dump,
+            instant_save,
+            only_mlist_urls,
+        )
+        return cls.from_mailing_lists(
+            name,
+            url_root,
+            lists,
+            select,
+            session,
+            only_mlist_urls,
         )
-        return cls.from_mailing_lists(name, url_root, lists, select, session)
 
     @classmethod
     def from_mailing_lists(
@@ -920,6 +964,7 @@ def from_mailing_lists(
         url_login: str = "https://list.etsi.org/scripts/wa.exe?LOGON",
         login: Optional[Dict[str, str]] = {"username": None, "password": None},
         session: Optional[str] = None,
+        only_mlist_urls: bool = True,
     ) -> "ListservArchive":
         """
         Create ListservArchive from a given list of 'ListservList'.
@@ -930,7 +975,7 @@ def from_mailing_lists(
             url_mailing_lists:
 
         """
-        if isinstance(url_mailing_lists[0], str):
+        if isinstance(url_mailing_lists[0], str) and only_mlist_urls is False:
             if session is None:
                 session = get_auth_session(url_login, **login)
             lists = []
@@ -987,8 +1032,9 @@ def get_lists_from_url(
         url_home: str,
         select: dict,
         session: Optional[str] = None,
-        instant_dump: bool = True,
-    ) -> List[ListservList]:
+        instant_save: bool = True,
+        only_mlist_urls: bool = True,
+    ) -> List[Union[ListservList, str]]:
         """
         Created dictionary of all lists in the archive.
 
@@ -1001,32 +1047,38 @@ def get_lists_from_url(
         # run through archive sections
         for url in list(
             ListservArchive.get_sections(url_root, url_home).keys()
-        )[:1]:
+        ):
             soup = get_website_content(url)
             a_tags_in_section = soup.select(
                 'a[href*="A0="][onmouseover*="showDesc"][onmouseout*="hideDesc"]',
             )
 
-            # run through archive lists in section
-            for a_tag in a_tags_in_section:
-                value = urllib.parse.urljoin(url_root, a_tag.get("href"))
-                key = value.split("A0=")[-1]
-                mlist = ListservList.from_url(
-                    name=key,
-                    url=value,
-                    select=select,
-                    session=session,
-                )
-                if len(mlist) != 0:
-                    if instant_dump:
-                        logger.info(
-                            f"The list {mlist.name} is save to a .mbox file."
-                        )
-                        mlist.to_mbox(dir_out=CONFIG.mail_path)
-                        archive.append(mlist.name)
-                    else:
-                        logger.info(f"Recorded the list {mlist.name}.")
-                        archive.append(mlist)
+            mlist_urls = [
+                urllib.parse.urljoin(url_root, a_tag.get("href"))
+                for a_tag in a_tags_in_section
+            ]
+
+            if only_mlist_urls:
+                # collect mailing-list urls
+                [archive.append(mlist_url) for mlist_url in mlist_urls]
+
+            else:
+                # collect mailing-list contents
+                for mlist_url in mlist_urls:
+                    key = mlist_url.split("A0=")[-1]
+                    mlist = ListservList.from_url(
+                        name=key,
+                        url=mlist_url,
+                        select=select,
+                        session=session,
+                    )
+                    if len(mlist) != 0:
+                        if instant_save:
+                            mlist.to_mbox(dir_out=CONFIG.mail_path)
+                            archive.append(mlist.name)
+                        else:
+                            logger.info(f"Recorded the list {mlist.name}.")
+                            archive.append(mlist)
         return archive
 
     def get_sections(url_root: str, url_home: str) -> int:
@@ -1092,16 +1144,6 @@ def to_dict(self) -> Dict[str, List[str]]:
                 .
                 "ListName": [messages[0], ... , messages[n]]
             }
-            {
-                "message_ID1":
-                    "body": ...,
-                    "subject": ...,
-                }
-                "message_ID2":
-                    "body": ...,
-                    "subject": ...,
-                }
-            }
         """
         # initialize dictionary
         dic = {}
diff --git a/examples/url_collections/listserv.3GPP.txt b/examples/url_collections/listserv.3GPP.txt
new file mode 100644
index 00000000..11d2952c
--- /dev/null
+++ b/examples/url_collections/listserv.3GPP.txt
@@ -0,0 +1,360 @@
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_CT_WG4_E_MEETING
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_CT_WG5
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_CT_WG6
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_CT_WG6_API_TEST
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_FUTURERAN
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_TDOC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG1
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG2
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG3
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG3_ELECTRONIC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG3_EMAIL_AGREEME
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG4
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG4_EDGE
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG4_GPRS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG4_PCS1900
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG4_TTCN
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_GERAN_WG5
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_LEADERS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_ADHOC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_AHG1
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_DRAFTS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_LTE_ADVANCED
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG1_EUL_AH
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG2_MBMS_RRC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG2_ROHC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG2_UMTS_DRAFTS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG3
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_AAS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_BANDS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_CA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_CPC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_DIVERSITY_OFF
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_DRAFTS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_EDCH
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_HSDPA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_IOT
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_LTE
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_MIMO-OTA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_NR-MIMO-OTA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_NR_BANDS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_TRP-TRS-OTA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG4_UE_ANTENNA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_5G_NR
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_ANALYTICS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_APPLICABILITY
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_EMEET
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_EMEET_RF
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_EMEET_SIG
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_FR2_MU
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_IMSCC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_IOT
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_LTE-CDMA2000
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_RF
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WG5_SIG
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_RAN_WGS_LONG_TERM_EVOLUTION
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_DRAFTS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG1_AIPN
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG1_GLOBALTEXT
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG1_VOCABULARY
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG1_WLAN
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2_DISCUSSIONS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2_ELCS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2_MTCE
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2_PROSE_GCSE_LTE
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2_UPCON
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG2_VCC
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4_EVS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4_EVS-IVAS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4_MBS
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4_MTSI
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4_SQ
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG4_VIDEO
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG5
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_SA_WG6
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_T_WG2_SWG1
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_TSG_T_WG2_SWG2
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_UE_POSITIONING
+https://list.etsi.org/scripts/wa.exe?A0=3GPP_WORKSHOP_LTE_GSM_HANDOVERS
+https://list.etsi.org/scripts/wa.exe?A0=462_ITS_VALID_CONF_FRMWK
+https://list.etsi.org/scripts/wa.exe?A0=AERO
+https://list.etsi.org/scripts/wa.exe?A0=AERO-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=AERO_A_SMGCS
+https://list.etsi.org/scripts/wa.exe?A0=AERO_DLS
+https://list.etsi.org/scripts/wa.exe?A0=AFI_ISG
+https://list.etsi.org/scripts/wa.exe?A0=AMETIS
+https://list.etsi.org/scripts/wa.exe?A0=ANSI_EU
+https://list.etsi.org/scripts/wa.exe?A0=ANTILOPE_ALL
+https://list.etsi.org/scripts/wa.exe?A0=ANTILOPE_COMMISSION
+https://list.etsi.org/scripts/wa.exe?A0=ANTILOPE_CORE
+https://list.etsi.org/scripts/wa.exe?A0=ANTILOPE_SEP
+https://list.etsi.org/scripts/wa.exe?A0=ANTILOPE_SVP
+https://list.etsi.org/scripts/wa.exe?A0=AT1
+https://list.etsi.org/scripts/wa.exe?A0=AT2
+https://list.etsi.org/scripts/wa.exe?A0=ATMOBILE
+https://list.etsi.org/scripts/wa.exe?A0=ATTM_AH_EETNT
+https://list.etsi.org/scripts/wa.exe?A0=B2B_WORKSHOP
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_CYSECNIS
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_ESSREV
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_M2M_PP
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_REGPOLES
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_RISE
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_SAP
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_SMRTCITY
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_SOOS
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_SSP
+https://list.etsi.org/scripts/wa.exe?A0=BOARD_TREND
+https://list.etsi.org/scripts/wa.exe?A0=BRAN_HM
+https://list.etsi.org/scripts/wa.exe?A0=BRAN_RCWG
+https://list.etsi.org/scripts/wa.exe?A0=BROADCAST_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=BROADCAST_M331
+https://list.etsi.org/scripts/wa.exe?A0=CENELEC_ETSI_DIGITAL_DIVIDEND
+https://list.etsi.org/scripts/wa.exe?A0=CENELEC_ETSI_JWG_EMC
+https://list.etsi.org/scripts/wa.exe?A0=CEPT_ETSI
+https://list.etsi.org/scripts/wa.exe?A0=CLC-ETSI-JWG-PLTCOEX
+https://list.etsi.org/scripts/wa.exe?A0=CLOUD
+https://list.etsi.org/scripts/wa.exe?A0=CLOUD_STANDARDS_COORDINATION
+https://list.etsi.org/scripts/wa.exe?A0=CLUBDEGOURMANDS
+https://list.etsi.org/scripts/wa.exe?A0=CSC_DATAPORT
+https://list.etsi.org/scripts/wa.exe?A0=CSC_INTEROP
+https://list.etsi.org/scripts/wa.exe?A0=CSC_REFERENCE
+https://list.etsi.org/scripts/wa.exe?A0=CSC_REVERSE
+https://list.etsi.org/scripts/wa.exe?A0=CSC_SECPRIV
+https://list.etsi.org/scripts/wa.exe?A0=CSC_SLA
+https://list.etsi.org/scripts/wa.exe?A0=CYBER-STF529-ABE-STEERING-GROUP
+https://list.etsi.org/scripts/wa.exe?A0=CYBER_RED_3DEF_SR
+https://list.etsi.org/scripts/wa.exe?A0=DECT_ADVANCED
+https://list.etsi.org/scripts/wa.exe?A0=DECT_INTEREST_GROUP
+https://list.etsi.org/scripts/wa.exe?A0=DECT_NEWGENERATION
+https://list.etsi.org/scripts/wa.exe?A0=DECT_URLLC
+https://list.etsi.org/scripts/wa.exe?A0=DELIVERABLE-NOTIFICATIONS
+https://list.etsi.org/scripts/wa.exe?A0=E-SIGNATURES_NEWS
+https://list.etsi.org/scripts/wa.exe?A0=E2NA
+https://list.etsi.org/scripts/wa.exe?A0=ECALL
+https://list.etsi.org/scripts/wa.exe?A0=EDS-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=EDS_PROMO_3GPP_COMMUNITY
+https://list.etsi.org/scripts/wa.exe?A0=EDS_PROMO_FORA
+https://list.etsi.org/scripts/wa.exe?A0=EDS_PROMO_GUESTS
+https://list.etsi.org/scripts/wa.exe?A0=EDS_PROMO_MESA_COMMUNITY
+https://list.etsi.org/scripts/wa.exe?A0=EDS_PROMO_MISC
+https://list.etsi.org/scripts/wa.exe?A0=EDS_PROMO_NO_MORE_CUSTOMER
+https://list.etsi.org/scripts/wa.exe?A0=EE2_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=EETNT
+https://list.etsi.org/scripts/wa.exe?A0=EE_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=EF3
+https://list.etsi.org/scripts/wa.exe?A0=EHEALTH_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=ERM_ACEA
+https://list.etsi.org/scripts/wa.exe?A0=ERM_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=ERM_COMBINEDE
+https://list.etsi.org/scripts/wa.exe?A0=ERM_EMC_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=ERM_FLX
+https://list.etsi.org/scripts/wa.exe?A0=ERM_ITU-R_WP8A
+https://list.etsi.org/scripts/wa.exe?A0=ERM_ITU-R_WP8A_P
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JOINT_TG34_TG28
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFEA
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFEA_A_SMGCS_HS_ASMR
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFEA_A_SMGCS_HS_MLAT
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFEA_GBAS
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFEA_HS_VDL_MODE2
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFEA_UAS
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JTFER
+https://list.etsi.org/scripts/wa.exe?A0=ERM_JWG_CLC_210
+https://list.etsi.org/scripts/wa.exe?A0=ERM_NEWS
+https://list.etsi.org/scripts/wa.exe?A0=ERM_RED
+https://list.etsi.org/scripts/wa.exe?A0=ERM_RM_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG14
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG17_WG1_2_4
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG21
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG23
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG25
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG25_A_CDM
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG25_A_SMGCS
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG25_DLS
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG31A
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG31C
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG37_DSRC
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG39
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TG41
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TGRX
+https://list.etsi.org/scripts/wa.exe?A0=ERM_TGUWB
+https://list.etsi.org/scripts/wa.exe?A0=ESI
+https://list.etsi.org/scripts/wa.exe?A0=ESI_ALGO
+https://list.etsi.org/scripts/wa.exe?A0=ESI_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=ESI_ASIG
+https://list.etsi.org/scripts/wa.exe?A0=ESI_CSP_CONFORMANCE
+https://list.etsi.org/scripts/wa.exe?A0=ESI_ERS
+https://list.etsi.org/scripts/wa.exe?A0=ESI_PSD2_WORKSHOP
+https://list.etsi.org/scripts/wa.exe?A0=ESI_SIG_CREA_VALID_TSP_SG
+https://list.etsi.org/scripts/wa.exe?A0=ESP_SUPPORT_TESTING
+https://list.etsi.org/scripts/wa.exe?A0=ETSI-EVENTS
+https://list.etsi.org/scripts/wa.exe?A0=ETSI-MEDIA-LIST
+https://list.etsi.org/scripts/wa.exe?A0=ETSI-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=ETSI_AIOTI_COORDINATION
+https://list.etsi.org/scripts/wa.exe?A0=ETSI_CDN
+https://list.etsi.org/scripts/wa.exe?A0=ETSI_M462
+https://list.etsi.org/scripts/wa.exe?A0=ETSI_M493
+https://list.etsi.org/scripts/wa.exe?A0=GA_ART-18_1
+https://list.etsi.org/scripts/wa.exe?A0=GSM_ONBOARD_PLANES
+https://list.etsi.org/scripts/wa.exe?A0=H2020_SMARTGRID_PROJECT_PROPOSAL
+https://list.etsi.org/scripts/wa.exe?A0=HF_MOBILE
+https://list.etsi.org/scripts/wa.exe?A0=HF_MULTICULTURAL_COMMUNICATION
+https://list.etsi.org/scripts/wa.exe?A0=HF_SETUP
+https://list.etsi.org/scripts/wa.exe?A0=HF_TELECAREUX
+https://list.etsi.org/scripts/wa.exe?A0=HIPERMAN_WIMAX_TESTING
+https://list.etsi.org/scripts/wa.exe?A0=HPC_DEG
+https://list.etsi.org/scripts/wa.exe?A0=IC-CG
+https://list.etsi.org/scripts/wa.exe?A0=INS_ISG
+https://list.etsi.org/scripts/wa.exe?A0=INT
+https://list.etsi.org/scripts/wa.exe?A0=IPR
+https://list.etsi.org/scripts/wa.exe?A0=ISG_CCM
+https://list.etsi.org/scripts/wa.exe?A0=ISG_CDP
+https://list.etsi.org/scripts/wa.exe?A0=ISG_ECI
+https://list.etsi.org/scripts/wa.exe?A0=ISG_IP6
+https://list.etsi.org/scripts/wa.exe?A0=ISG_MBC
+https://list.etsi.org/scripts/wa.exe?A0=ISG_MEC_IEG
+https://list.etsi.org/scripts/wa.exe?A0=ISG_QSC
+https://list.etsi.org/scripts/wa.exe?A0=ISG_QSC_WI01
+https://list.etsi.org/scripts/wa.exe?A0=ISG_QSC_WI02
+https://list.etsi.org/scripts/wa.exe?A0=ISG_QSC_WI03
+https://list.etsi.org/scripts/wa.exe?A0=ISG_QSC_WI04
+https://list.etsi.org/scripts/wa.exe?A0=ISG_QSC_WI05
+https://list.etsi.org/scripts/wa.exe?A0=ISI_ISG
+https://list.etsi.org/scripts/wa.exe?A0=ITS
+https://list.etsi.org/scripts/wa.exe?A0=ITS-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=ITS-SPECTRUM
+https://list.etsi.org/scripts/wa.exe?A0=ITS_WG1
+https://list.etsi.org/scripts/wa.exe?A0=ITS_WG2
+https://list.etsi.org/scripts/wa.exe?A0=ITS_WG3
+https://list.etsi.org/scripts/wa.exe?A0=ITS_WG4
+https://list.etsi.org/scripts/wa.exe?A0=ITS_WG5
+https://list.etsi.org/scripts/wa.exe?A0=LI-LTE-FAQ-GROUP
+https://list.etsi.org/scripts/wa.exe?A0=LI-NFV-AD-HOC-GROUP
+https://list.etsi.org/scripts/wa.exe?A0=LIS_ISG
+https://list.etsi.org/scripts/wa.exe?A0=LI_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=LTN_ISG
+https://list.etsi.org/scripts/wa.exe?A0=LTS
+https://list.etsi.org/scripts/wa.exe?A0=M2M2014WS_PC
+https://list.etsi.org/scripts/wa.exe?A0=M2M_RAPPORTEURS
+https://list.etsi.org/scripts/wa.exe?A0=M436_RFID
+https://list.etsi.org/scripts/wa.exe?A0=M436_RFID_CG
+https://list.etsi.org/scripts/wa.exe?A0=M436_RFID_CONSULTATION
+https://list.etsi.org/scripts/wa.exe?A0=MARITIME-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=MARITIME_NEWS
+https://list.etsi.org/scripts/wa.exe?A0=MCD
+https://list.etsi.org/scripts/wa.exe?A0=MCD_CA_DRM_INTEROP
+https://list.etsi.org/scripts/wa.exe?A0=MCD_CDN-I
+https://list.etsi.org/scripts/wa.exe?A0=MCD_INTEREST_GROUP
+https://list.etsi.org/scripts/wa.exe?A0=MCD_MANAGE
+https://list.etsi.org/scripts/wa.exe?A0=MESA_ORGANIZATION
+https://list.etsi.org/scripts/wa.exe?A0=MESA_PS_MEM
+https://list.etsi.org/scripts/wa.exe?A0=MESA_SC
+https://list.etsi.org/scripts/wa.exe?A0=MESA_SSG_SA
+https://list.etsi.org/scripts/wa.exe?A0=MESA_TSG_SYS
+https://list.etsi.org/scripts/wa.exe?A0=MOI_ISG
+https://list.etsi.org/scripts/wa.exe?A0=MSG_TFES_BS_DD
+https://list.etsi.org/scripts/wa.exe?A0=MSG_TFES_UE_DD
+https://list.etsi.org/scripts/wa.exe?A0=MTC
+https://list.etsi.org/scripts/wa.exe?A0=MTC_ISG
+https://list.etsi.org/scripts/wa.exe?A0=NFV_AVINF_WG
+https://list.etsi.org/scripts/wa.exe?A0=NFV_E2E_ARCH
+https://list.etsi.org/scripts/wa.exe?A0=NFV_MANO_WG
+https://list.etsi.org/scripts/wa.exe?A0=NFV_NOC_SMALLTEAM
+https://list.etsi.org/scripts/wa.exe?A0=NFV_PER_EG
+https://list.etsi.org/scripts/wa.exe?A0=NFV_PHASE2
+https://list.etsi.org/scripts/wa.exe?A0=NFV_REQ
+https://list.etsi.org/scripts/wa.exe?A0=NFV_SW_ARCH_WG
+https://list.etsi.org/scripts/wa.exe?A0=NFV_USE_CASES
+https://list.etsi.org/scripts/wa.exe?A0=NGN_IMS
+https://list.etsi.org/scripts/wa.exe?A0=NGN_INTERCONNECT
+https://list.etsi.org/scripts/wa.exe?A0=NSO-CONTACTS
+https://list.etsi.org/scripts/wa.exe?A0=NTECH_AFI
+https://list.etsi.org/scripts/wa.exe?A0=OCG_AI
+https://list.etsi.org/scripts/wa.exe?A0=OCG_AN
+https://list.etsi.org/scripts/wa.exe?A0=OCG_MCP
+https://list.etsi.org/scripts/wa.exe?A0=OCG_RED
+https://list.etsi.org/scripts/wa.exe?A0=OCG_SECURITY
+https://list.etsi.org/scripts/wa.exe?A0=ORI_ISG
+https://list.etsi.org/scripts/wa.exe?A0=OSG_ISG
+https://list.etsi.org/scripts/wa.exe?A0=PLT
+https://list.etsi.org/scripts/wa.exe?A0=PLT_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=PLT_INHOUSE
+https://list.etsi.org/scripts/wa.exe?A0=PLT_WI_DISCUSSION
+https://list.etsi.org/scripts/wa.exe?A0=PLU_RFID_CHINA
+https://list.etsi.org/scripts/wa.exe?A0=PORTAL-FEATURE-REQUESTS
+https://list.etsi.org/scripts/wa.exe?A0=QKD_ISG
+https://list.etsi.org/scripts/wa.exe?A0=RAIL-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=RRS_WG4
+https://list.etsi.org/scripts/wa.exe?A0=RT
+https://list.etsi.org/scripts/wa.exe?A0=RT_DMO
+https://list.etsi.org/scripts/wa.exe?A0=RT_NG2R
+https://list.etsi.org/scripts/wa.exe?A0=SES_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=SES_BSM
+https://list.etsi.org/scripts/wa.exe?A0=SES_BSM_C2P
+https://list.etsi.org/scripts/wa.exe?A0=SES_ECAS
+https://list.etsi.org/scripts/wa.exe?A0=SES_GMR_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=SES_MSS
+https://list.etsi.org/scripts/wa.exe?A0=SES_SDR
+https://list.etsi.org/scripts/wa.exe?A0=SES_SUMTS_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=SG_GNGN
+https://list.etsi.org/scripts/wa.exe?A0=SG_UCI
+https://list.etsi.org/scripts/wa.exe?A0=SMG_PLENARY
+https://list.etsi.org/scripts/wa.exe?A0=SMT_ISG
+https://list.etsi.org/scripts/wa.exe?A0=SPR-NEWS
+https://list.etsi.org/scripts/wa.exe?A0=STF496_ERMTG26_RED
+https://list.etsi.org/scripts/wa.exe?A0=STF504_STQ_EMODET
+https://list.etsi.org/scripts/wa.exe?A0=STF510_SMARTM2M_SAP_TEST
+https://list.etsi.org/scripts/wa.exe?A0=STF514
+https://list.etsi.org/scripts/wa.exe?A0=STF517_ITS_VALIDATION
+https://list.etsi.org/scripts/wa.exe?A0=STF518
+https://list.etsi.org/scripts/wa.exe?A0=STF519_INT_S1AP_TEST
+https://list.etsi.org/scripts/wa.exe?A0=STF525_INTEROP_PLATFORM
+https://list.etsi.org/scripts/wa.exe?A0=STF527_TEST_GEONW
+https://list.etsi.org/scripts/wa.exe?A0=STF528_DECT
+https://list.etsi.org/scripts/wa.exe?A0=STF530_NFV
+https://list.etsi.org/scripts/wa.exe?A0=STF533
+https://list.etsi.org/scripts/wa.exe?A0=STF536_REVISION_EN_301549
+https://list.etsi.org/scripts/wa.exe?A0=STQ
+https://list.etsi.org/scripts/wa.exe?A0=STQ_AURORA
+https://list.etsi.org/scripts/wa.exe?A0=SWCOPYRIGHT
+https://list.etsi.org/scripts/wa.exe?A0=TESTCOM_2003_TPC
+https://list.etsi.org/scripts/wa.exe?A0=TEST_DB
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG1_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG38
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG3_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG4_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG5_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG6_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRAWG8_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRA_ANNOUNCE
+https://list.etsi.org/scripts/wa.exe?A0=TETRA_WORKSHOP
+https://list.etsi.org/scripts/wa.exe?A0=TEXEL
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_EMTEL
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_F-MMS
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_FORUM
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_FUTURE_ACTIVITIES
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_GEN
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_NGN
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_STF328_SG
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG1
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG2
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG2_IPTV
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG3
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG3_IPTV_STAGE3
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG4
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG5
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG5_OLD
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG6
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG7
+https://list.etsi.org/scripts/wa.exe?A0=TISPAN_WG8
+https://list.etsi.org/scripts/wa.exe?A0=TM1-WP1
+https://list.etsi.org/scripts/wa.exe?A0=TM1-WP2
+https://list.etsi.org/scripts/wa.exe?A0=TM6_ALL
+https://list.etsi.org/scripts/wa.exe?A0=TMSIX_OPS
+https://list.etsi.org/scripts/wa.exe?A0=USER_ANNOUNCE


Test patch:
diff --git a/tests/webscraping/test_listserv.py b/tests/webscraping/test_listserv.py
index b21e90fd..6a8de24e 100644
--- a/tests/webscraping/test_listserv.py
+++ b/tests/webscraping/test_listserv.py
@@ -177,7 +177,8 @@ def test__from_url_with_login(self):
                 "fields": "header",
             },
             login=auth_key,
-            instant_dump=False,
+            instant_save=False,
+            only_mlist_urls=False,
         )
         assert (
             arch.lists[0].messages[0].fromaddr == "Kimmo.Kymalainen@ETSI.ORG"
@@ -197,26 +198,29 @@ def get_mailarchive(self):
                 "fields": "header",
             },
             login=auth_key_mock,
-            instant_dump=False,
+            instant_save=False,
+            only_mlist_urls=False,
         )
         return arch
 
     def test__archive_content(self, arch):
+        mlist_names = [mlist.name for mlist in arch.lists]
+        mlist_len = [len(mlist) for mlist in arch.lists]
         assert arch.name == "3GPP"
         assert arch.url == url_archive
-        assert len(arch) == 4
+        assert len(arch) == 13
         assert len(arch.lists[0]) == 3
         assert arch.lists[0].messages[0].subject == "Happy New Year 2021"
 
     def test__to_dict(self, arch):
         dic = arch.to_dict()
         assert len(list(dic.keys())) == 9
-        assert len(dic[list(dic.keys())[0]]) == 40
+        assert len(dic[list(dic.keys())[0]]) == 221
 
     def test__to_pandas_dataframe(self, arch):
         df = arch.to_pandas_dataframe()
         assert len(df.columns.values) == 9
-        assert len(df.index.values) == 40
+        assert len(df.index.values) == 221
 
     def test__to_mbox(self, arch):
         arch.to_mbox(dir_temp)