Repository: datactive/bigbang (language: python)

Issue:
combine load_data and open_list_archives ? 
https://github.com/datactive/bigbang/pull/500/files#r753826294


Discussion / hints:
> Reading over this code again, it seems like load_data and open_list_archives actually perform the same basic task, just that load_data only opens CSV files and open_list_archives looks for and opens MBOX files with various filenames. The mbox parameter in both doesn't seem to do much (for load_data it switches to open_list_archives and for open_list_archives it's just a flag for interpreting the path parameter as a single file name).

> Could we have a single function that loads the already-collected mailing list archives for a particular list and returns it as a dataframe? And then internally bigbang can detect whether it's stored as a csv or a bunch of mbox files?

How is the LISTSERV data stored locally after it has been collected, @Christovis ?

Does your LISTSERV code use either of these methods, or does it use its own alternative version of them?

Does the W3C scraper use mbox or csv format, finally?

Maybe we should settle on a canonical data format for email archives of all kinds.

Or, otherwise, we should maybe separate the storage of "raw" email data, when available, and the nicely preprocessed into schematized CSV format that we support deeper analysis on.

Fix patch:
diff --git a/bigbang/__init__.py b/bigbang/__init__.py
index 29275d29..e63238c9 100644
--- a/bigbang/__init__.py
+++ b/bigbang/__init__.py
@@ -1,2 +1,4 @@
+from . import ingress
 from . import analysis
 from . import visualisation
+from . import bigbang_io
diff --git a/bigbang/analysis/listserv.py b/bigbang/analysis/listserv.py
index 568582f2..8a7af45a 100644
--- a/bigbang/analysis/listserv.py
+++ b/bigbang/analysis/listserv.py
@@ -21,11 +21,7 @@
 
 from config.config import CONFIG
 
-from bigbang.bigbang_io import (
-    ListservMessageIO,
-    ListservListIO,
-    ListservArchiveIO,
-)
+import bigbang.bigbang_io as bio
 from bigbang.utils import (
     get_paths_to_files_in_directory,
     get_paths_to_dirs_in_directory,
@@ -59,19 +55,19 @@
 # =?utf-8?b?J++9llNSVkND?=
 
 
-class ListservListWarning(BaseException):
-    """Base class for Archive class specific exceptions"""
+class ListservMailListWarning(BaseException):
+    """Base class for ListservMailList class specific exceptions"""
 
     pass
 
 
-class ListservArchiveWarning(BaseException):
-    """Base class for Archive class specific exceptions"""
+class ListservMailListDomainWarning(BaseException):
+    """Base class for ListservMailListDomain class specific exceptions"""
 
     pass
 
 
-class ListservList:
+class ListservMailList:
     """
     Note
     ----
@@ -129,8 +125,8 @@ def from_mbox(
         name: str,
         filepath: str,
         include_body: bool = True,
-    ) -> "ListservList":
-        df = ListservListIO.from_mbox_to_pandas_dataframe(filepath)
+    ) -> "ListservMailList":
+        df = bio.mlist_from_mbox_to_pandas_dataframe(filepath)
         return cls.from_pandas_dataframe(df, name, filepath)
 
     @classmethod
@@ -139,7 +135,7 @@ def from_pandas_dataframe(
         df: pd.DataFrame,
         name: Optional[str] = None,
         filepath: Optional[str] = None,
-    ) -> "ListservList":
+    ) -> "ListservMailList":
         return cls(name, filepath, df)
 
     @staticmethod
@@ -164,10 +160,15 @@ def contract(count: np.array, label: list, contract: float) -> dict:
     @staticmethod
     def get_name_localpart_domain(string: str) -> tuple:
         """
-        Split an address field which has a format as
-        'Heinrich von Kleist <Heinrich.vonKleist@SELBST.ORG>' into name,
+        Split an address field which has (ideally) a format as
+        'Heinrich von Kleist <Heinrich.vonKleist@SELBST.org>' into name,
         local-part, and domain.
         All strings are returned in lower case only to avoid duplicates.
+
+        Note
+        ----
+        Test whether the incorporation of email.utils.parseaddr() can improve
+        this function.
         """
         # test if name is in string or if address is duplicated
         test = string.split(" ")
@@ -176,10 +177,14 @@ def get_name_localpart_domain(string: str) -> tuple:
             and all("@" in st for st in test)
             and (test[0] == test[1])
         ):
+            # identifies addresses of the form:
+            # localpart@domain localpart@domain
             addr = test[0]
             localpart, domain = addr.split("@")
             return None, localpart.lower(), domain.lower()
-        elif (len(test) > 2) and "@" in test[-1]:
+        elif (len(test) > 2) and ("@" in test[-1]):
+            # identifies addresses of the form:
+            # abc def ghi localpart@domain
             name = (" ").join(test[:-1])
             name = name.replace('"', "").replace('"', "")
             addr = test[-1]
@@ -187,8 +192,12 @@ def get_name_localpart_domain(string: str) -> tuple:
             localpart, domain = addr.split("@")
             return name.lower(), localpart.lower(), domain.lower()
         else:
-            name, addr = email.utils.parseaddr(string)
-            addr = addr.split(" ")[-1]
+            # identifies addresses of the form:
+            # abc localpart@domain
+            name = string.split(" ")[0]
+            name = name.replace('"', "")
+            addr = string.split(" ")[-1]
+            addr = addr.replace("<", "").replace(">", "")
             if "@" in addr:
                 localpart, domain = addr.split("@")
                 return name.lower(), localpart.lower(), domain.lower()
@@ -203,12 +212,16 @@ def get_name_localpart_domain(string: str) -> tuple:
     def iterator_name_localpart_domain(li: list) -> tuple:
         """Generator for the self.get_name_localpart_domain() function."""
         for string in li:
-            if ("<" in string) and (">" in string):
+            # identify whether there are multiple addresses in one header field
+            if (string.count("<") > 1) and (string.count(">") > 1):
                 addresses = re.findall(r"(?<=\<).+?(?=\>)", string)
                 for addr in addresses:
-                    yield ListservList.get_name_localpart_domain(addr)
+                    if "@" in addr:
+                        yield ListservMailList.get_name_localpart_domain(addr)
+                    else:
+                        continue
             else:
-                yield ListservList.get_name_localpart_domain(string)
+                yield ListservMailList.get_name_localpart_domain(string)
 
     def period_of_activity(self) -> list:
         """
@@ -222,7 +235,7 @@ def period_of_activity(self) -> list:
         ]
         return period_of_activity
 
-    def crop_by_year(self, yrs: Union[int, list]) -> "ListservList":
+    def crop_by_year(self, yrs: Union[int, list]) -> "ListservMailList":
         """
         Filter `self.df` DataFrame by year in message date.
 
@@ -233,7 +246,7 @@ def crop_by_year(self, yrs: Union[int, list]) -> "ListservList":
 
         Returns
         -------
-        `ListservList` object cropped to specification.
+        `ListservMailList` object cropped to specification.
         """
         index = get_index_of_msgs_with_datetime(self.df)
         _df = self.df.loc[index]
@@ -244,7 +257,7 @@ def crop_by_year(self, yrs: Union[int, list]) -> "ListservList":
                 (dt.year >= min(yrs)) & (dt.year < max([yrs]))
                 for dt in _df["date"].values
             ]
-        return ListservList.from_pandas_dataframe(
+        return ListservMailList.from_pandas_dataframe(
             df=_df.loc[mask],
             name=self.name,
             filepath=self.filepath,
@@ -254,7 +267,7 @@ def crop_by_address(
         self,
         header_field: str,
         per_address_field: Dict[str, List[str]],
-    ) -> "ListservList":
+    ) -> "ListservMailList":
         """
         Parameters
         ----------
@@ -267,13 +280,13 @@ def crop_by_address(
 
         Returns
         -------
-        `ListservList` object cropped to specification.
+        `ListservMailList` object cropped to specification.
         """
         mlist = self.df.dropna(subset=["from"])
         if "domain" in list(per_address_field.keys()):
             _addr = pd.Series(
                 [
-                    ListservList.get_name_localpart_domain(addr)[-1]
+                    ListservMailList.get_name_localpart_domain(addr)[-1]
                     for addr in mlist["from"].values
                 ],
                 index=mlist.index.values,
@@ -283,20 +296,20 @@ def crop_by_address(
         if "localpart" in list(per_address_field.keys()):
             _addr = pd.Series(
                 [
-                    ListservList.get_name_localpart_domain(addr)[1]
+                    ListservMailList.get_name_localpart_domain(addr)[1]
                     for addr in mlist["from"].values
                 ],
                 index=mlist.index.values,
             ).dropna()
             mlist = mlist.loc[_addr.index]
             mlist = mlist[_addr.isin(per_address_field["localpart"])]
-        return ListservList.from_pandas_dataframe(
+        return ListservMailList.from_pandas_dataframe(
             df=mlist,
             name=self.name,
             filepath=self.filepath,
         )
 
-    def crop_by_subject(self, match=str, place: int = 2) -> "ListservList":
+    def crop_by_subject(self, match=str, place: int = 2) -> "ListservMailList":
         """
         Parameters
         ----------
@@ -308,7 +321,7 @@ def crop_by_subject(self, match=str, place: int = 2) -> "ListservList":
 
         Returns
         -------
-        `ListservList` object cropped to message subject.
+        `ListservMailList` object cropped to message subject.
         """
         index = get_index_of_msgs_with_subject(self.df)
         _df = self.df.loc[index]
@@ -333,7 +346,7 @@ def crop_by_subject(self, match=str, place: int = 2) -> "ListservList":
                 else False
             )
         mask = _df["subject"].apply(func).values  # returns bool-type array
-        return ListservList.from_pandas_dataframe(
+        return ListservMailList.from_pandas_dataframe(
             df=_df.loc[mask],
             name=self.name,
             filepath=self.filepath,
@@ -365,7 +378,7 @@ def get_domains(
             df = self.df
         domains = {}
         for header_field in header_fields:
-            generator = ListservList.iterator_name_localpart_domain(
+            generator = ListservMailList.iterator_name_localpart_domain(
                 df[header_field].dropna().values
             )
             # collect domain labels
@@ -443,7 +456,7 @@ def get_localparts(
             if per_domain:
                 # TODO: Needs two for loop. Find way to reduce it.
                 localparts[header_field] = {}
-                generator = ListservList.iterator_name_localpart_domain(
+                generator = ListservMailList.iterator_name_localpart_domain(
                     df[header_field].dropna().values
                 )
                 _domains = list(
@@ -456,7 +469,7 @@ def get_localparts(
                     )
                 )
                 _localparts = {d: [] for d in _domains}
-                generator = ListservList.iterator_name_localpart_domain(
+                generator = ListservMailList.iterator_name_localpart_domain(
                     df[header_field].dropna().values
                 )
                 for _, localpart, domain in generator:
@@ -469,7 +482,7 @@ def get_localparts(
                     _localparts[_domain] = _dolpsu
                 localparts[header_field] = _localparts
             else:
-                generator = ListservList.iterator_name_localpart_domain(
+                generator = ListservMailList.iterator_name_localpart_domain(
                     df[header_field].dropna().values
                 )
                 _localparts = [
@@ -653,7 +666,7 @@ def get_threadsroot(
         }
         # sort into address field
         if per_address_field:
-            generator = ListservList.iterator_name_localpart_domain(
+            generator = ListservMailList.iterator_name_localpart_domain(
                 _df["from"].values
             )
             if "domain" in per_address_field:
@@ -839,13 +852,15 @@ def get_sender_receiver_dict(
             dic = {lp: {} for lp in localparts}
         # loop through messages
         for index, row in df.iterrows():
-            _, f_localpart, f_domain = ListservList.get_name_localpart_domain(
-                row["from"]
-            )
+            (
+                _,
+                f_localpart,
+                f_domain,
+            ) = ListservMailList.get_name_localpart_domain(row["from"])
             if f_domain is None:
                 continue
 
-            ct_generator = ListservList.iterator_name_localpart_domain(
+            ct_generator = ListservMailList.iterator_name_localpart_domain(
                 [df.loc[index]["comments-to"]]
             )
             for _, ct_localpart, ct_domain in ct_generator:
@@ -910,7 +925,7 @@ def create_sender_receiver_digraph(
     ):
         """
         Create directed graph from messaging network created with
-        ListservList.get_sender_receiver_dict().
+        ListservMailList.get_sender_receiver_dict().
 
         Parameters
         ----------
@@ -1012,16 +1027,16 @@ def get_graph_prop_per_domain_per_year(
         return dic_evol
 
 
-class ListservArchive:
+class ListservMailListDomain:
     """
     Parameters
     ----------
     name
-        The of whom the archive is (e.g. 3GPP, IEEE, ...)
+        The of whom the mail list domain is (e.g. 3GPP, IEEE, ...)
     filedsc
-        The file description of the archive
+        The file description of the mail list domain
     lists
-        A list containing the mailing lists as `ListservList` types
+        A list containing the mailing lists as `ListservMailList` types
 
     Methods
     -------
@@ -1039,13 +1054,15 @@ def from_mbox(
         name: str,
         directorypath: str,
         filedsc: str = "*.mbox",
-    ) -> "ListservArchive":
+    ) -> "ListservMailListDomain":
         filepaths = get_paths_to_files_in_directory(directorypath, filedsc)
         if len(filepaths) > 0:
-            ListservArchiveWarning("No files found fitting the description")
+            ListservMailListDomainWarning(
+                "No files found fitting the description"
+            )
         for count, filepath in enumerate(filepaths):
             name = filepath.split("/")[-1].split(".")[0]
-            mlist = ListservList.from_mbox(name, filepath).df
+            mlist = ListservMailList.from_mbox(name, filepath).df
             mlist["mailing-list"] = name
             if count == 0:
                 mlists = mlist
diff --git a/bigbang/bigbang_io.py b/bigbang/bigbang_io.py
index 006094da..1faf6a2f 100644
--- a/bigbang/bigbang_io.py
+++ b/bigbang/bigbang_io.py
@@ -1,7 +1,3 @@
-"""
-Input/Output for Listserv data.
-"""
-
 import datetime
 import glob
 import os
@@ -16,6 +12,7 @@
 from config.config import CONFIG
 
 from bigbang.analysis import utils
+from bigbang.data_types import Message, MailList, MailListDomain
 
 filepath_auth = CONFIG.config_path + "authentication.yaml"
 directory_project = str(Path(os.path.abspath(__file__)).parent.parent)
@@ -28,209 +25,230 @@
 logger = logging.getLogger(__name__)
 
 
-class ListservMessageIO:
+def email_to_dict(msg: Message) -> Dict[str, str]:
     """
-    Methods
-    -------
-    to_dict
-    to_pandas_dataframe
-    to_mbox
+    Handles data type transformation from `mailbox.mboxMessage` to `Dictionary`.
     """
+    dic = {"Body": msg.get_payload()}
+    for key, value in msg.items():
+        dic[key] = value
+    return dic
 
-    def to_dict(msg: mboxMessage) -> Dict[str, str]:
-        dic = {"Body": msg.get_payload()}
-        for key, value in msg.items():
-            dic[key] = value
-        return dic
 
-    def to_pandas_dataframe(msg: mboxMessage) -> pd.DataFrame:
-        return pd.DataFrame(
-            ListservMessageIO.to_dict(msg),
-            index=[msg["Message-ID"]],
-        )
+def email_to_pandas_dataframe(msg: Message) -> pd.DataFrame:
+    """
+    Handles data type transformation from `mailbox.mboxMessage` to `pandas.DataFrame`.
+    """
+    return pd.DataFrame(
+        email_to_dict(msg),
+        index=[msg["Message-ID"]],
+    )
+
 
-    def to_mbox(msg: mboxMessage, filepath: str, mode: str = "w"):
-        """
-        Safe message to .mbox file.
-        """
-        if Path(filepath).is_file():
-            Path(filepath).unlink()
-        mbox = mailbox.mbox(filepath)
-        mbox.lock()
-        mbox.add(msg)
-        mbox.flush()
-        mbox.unlock()
-
-
-class ListservListIO:
-    """
-    This class handles the data transformations for Listserv Lists.
-    """
-
-    def from_mbox_to_pandas_dataframe(filepath: str) -> pd.DataFrame:
-        box = mailbox.mbox(filepath, create=False)
-        mlist = [dict(msg) for msg in list(box.values())]
-        # TODO: find out why some fields are missing in some messages
-        nr_of_fields = np.array([len(msg.keys()) for msg in mlist])
-        nr_of_fields = np.unique(nr_of_fields, return_counts=True)
-        df = pd.DataFrame(mlist)
-        df = utils.clean_addresses(df)
-        df = utils.clean_subject(df)
-        df = utils.clean_datetime(df)
-        return df
-
-    def from_mbox(filepath: str) -> list:
-        """
-        Parameters
-        ----------
-        name : Name of the mailing list, e.g., '3GPP_TSG_RAN_WG3'
-        filepath : Path to the file, e.g.,
-            '/home/rumpelstielzchen/bigbang/archive/3GPP/3GPP_TSG_RAN_WG3.mbox'
-        """
-        box = mailbox.mbox(filepath, create=False)
-        return list(box.values())
-
-    def to_dict(msgs: list, include_body: bool = True) -> Dict[str, List[str]]:
-        """
-        Place all message into a dictionary.
-        """
-        dic = {}
+def email_to_mbox(msg: Message, filepath: str, mode: str = "w") -> None:
+    """
+    Saves `mailbox.mboxMessage` as .mbox file.
+    """
+    if Path(filepath).is_file():
+        Path(filepath).unlink()
+    mbox = mailbox.mbox(filepath)
+    mbox.lock()
+    mbox.add(msg)
+    mbox.flush()
+    mbox.unlock()
+
+
+def mlist_from_mbox_to_pandas_dataframe(filepath: str) -> pd.DataFrame:
+    """
+    Reads `mailbox.mboxMessage` objects from .mbox file and transforms it to a
+    `pandas.DataFrame`.
+    For a clearer definition on what a mailing list is, see:
+    bigbang.ingress.abstract.AbstractList
+    """
+    box = mailbox.mbox(filepath, create=False)
+    mlist = [dict(msg) for msg in list(box.values())]
+    # TODO: find out why some fields are missing in some messages
+    nr_of_fields = np.array([len(msg.keys()) for msg in mlist])
+    nr_of_fields = np.unique(nr_of_fields, return_counts=True)
+    df = pd.DataFrame(mlist)
+    df = utils.clean_addresses(df)
+    df = utils.clean_subject(df)
+    df = utils.clean_datetime(df)
+    return df
+
+
+def mlist_from_mbox(filepath: str) -> list:
+    """
+    Reads `mailbox.mboxMessage` objects from .mbox file.
+    For a clearer definition on what a mailing list is, see:
+    bigbang.ingress.abstract.AbstractList
+    """
+    box = mailbox.mbox(filepath, create=False)
+    return list(box.values())
+
+
+def mlist_to_dict(
+    msgs: MailList,
+    include_body: bool = True,
+) -> Dict[str, List[str]]:
+    """
+    Handles data type transformation from a List[mailbox.mboxMessage] to a
+    Dictionary.
+    For a clearer definition on what a mailing list is, see:
+    bigbang.ingress.abstract.AbstractList
+    """
+    dic = {}
+    for idx, msg in enumerate(msgs):
+        # if msg["message-id"] != None:  #TODO: why are some 'None'?
+        for key, value in msg.items():
+            key = key.lower()
+            if key not in dic.keys():
+                dic[key] = [np.nan] * len(msgs)
+            dic[key][idx] = value
+    if include_body:
+        dic["body"] = [np.nan] * len(msgs)
         for idx, msg in enumerate(msgs):
-            # if msg["message-id"] != None:  #TODO: why are some 'None'?
-            for key, value in msg.items():
-                key = key.lower()
-                if key not in dic.keys():
-                    dic[key] = [np.nan] * len(msgs)
-                dic[key][idx] = value
-        if include_body:
-            dic["body"] = [np.nan] * len(msgs)
-            for idx, msg in enumerate(msgs):
-                # if msg["message-id"] != None:
-                dic["body"][idx] = msg.get_payload()
-        lengths = [len(value) for value in dic.values()]
-        assert all([diff == 0 for diff in np.diff(lengths)])
-        return dic
-
-    def to_pandas_dataframe(
-        msgs: list, include_body: bool = True
-    ) -> pd.DataFrame:
-        dic = ListservListIO.to_dict(msgs, include_body)
-        df = pd.DataFrame(dic)
-        # filter out messages with unrecognisable datetime
-        index = np.array(
-            [
-                True
-                if (isinstance(dt, str)) and (len(dt) > 10) and not pd.isna(dt)
-                else False
-                for i, dt in enumerate(df["date"])
-            ],
-            dtype="bool",
-        )
-        df = df.loc[index, :]
-        # convert data type from string to datetime.datetime object
-        df.loc[:, "date"].update(
-            df.loc[:, "date"].apply(
-                lambda x: datetime.datetime.strptime(
-                    x, "%a, %d %b %Y %H:%M:%S %z"
-                )
-            )
-        )
-        return df
-
-    def to_mbox(msgs: list, dir_out: str, filename: str):
-        """
-        Safe mailing list to .mbox files.
-        """
-        # create filepath
-        filepath = f"{dir_out}/{filename}.mbox"
-        # delete file if there is one at the filepath
-        if Path(filepath).is_file():
-            Path(filepath).unlink()
-        mbox = mailbox.mbox(filepath)
-        mbox.lock()
-        for msg in msgs:
-            try:
-                mbox.add(msg)
-            except Exception as e:
-                logger.info(
-                    f'Add to .mbox error for {msg["Archived-At"]} because, {e}'
-                )
-        mbox.flush()
-        mbox.unlock()
-        logger.info(f"The list {filename} is saved at {filepath}.")
-        mbox.lock()
-        mbox.add(msg)
-        mbox.flush()
-        mbox.unlock()
-
-
-class ListservArchiveIO:
-    """
-    This class handles the data transformations for Listserv Archives.
-    """
-
-    def to_dict(
-        mlists: list, include_body: bool = True
-    ) -> Dict[str, List[str]]:
-        """
-        Place all message into a dictionary.
-        """
-        nr_msgs = 0
-        for ii, mlist in enumerate(mlists):
-            dic_mlist = ListservListIO.to_dict(mlist.messages, include_body)
-            if ii == 0:
-                dic_march = dic_mlist
-                dic_march["mailing-list"] = [mlist.name] * len(mlist)
-            else:
-                # add mlist items to march
-                for key, value in dic_mlist.items():
-                    if key not in dic_march.keys():
-                        dic_march[key] = [np.nan] * nr_msgs
-                    dic_march[key].extend(value)
-                # if mlist does not contain items that are in march
-                key_miss = list(set(dic_march.keys()) - set(dic_mlist.keys()))
-                key_miss.remove("mailing-list")
-                for key in key_miss:
-                    dic_march[key].extend([np.nan] * len(mlist))
-
-                dic_march["mailing-list"].extend([mlist.name] * len(mlist))
-            nr_msgs += len(mlist)
-        lengths = [len(value) for value in dic_march.values()]
-        assert all([diff == 0 for diff in np.diff(lengths)])
-        return dic_march
-
-    def to_pandas_dataframe(
-        mlists: list, include_body: bool = True
-    ) -> pd.DataFrame:
-        df = pd.DataFrame(
-            ListservArchiveIO.to_dict(mlists, include_body)
-        ).set_index("message-id")
-        # get index of date-times
-        index = np.array(
-            [
-                True
-                if (isinstance(dt, str)) and (len(dt) > 10) and not pd.isna(dt)
-                else False
-                for i, dt in enumerate(df["date"])
-            ],
-            dtype="bool",
+            # if msg["message-id"] != None:
+            dic["body"][idx] = msg.get_payload()
+    lengths = [len(value) for value in dic.values()]
+    assert all([diff == 0 for diff in np.diff(lengths)])
+    return dic
+
+
+def mlist_to_pandas_dataframe(
+    msgs: MailList,
+    include_body: bool = True,
+) -> pd.DataFrame:
+    """
+    Handles data type transformation from a List[mailbox.mboxMessage] to a
+    pandas.DataFrame.
+    For a clearer definition on what a mailing list is, see:
+    bigbang.ingress.abstract.AbstractList
+    """
+    dic = mlist_to_dict(msgs, include_body)
+    df = pd.DataFrame(dic)
+    # filter out messages with unrecognisable datetime
+    index = np.array(
+        [
+            True
+            if (isinstance(dt, str)) and (len(dt) > 10) and not pd.isna(dt)
+            else False
+            for i, dt in enumerate(df["date"])
+        ],
+        dtype="bool",
+    )
+    df = df.loc[index, :]
+    # convert data type from string to datetime.datetime object
+    df.loc[:, "date"].update(
+        df.loc[:, "date"].apply(
+            lambda x: datetime.datetime.strptime(x, "%a, %d %b %Y %H:%M:%S %z")
         )
-        # convert data type from string to datetime.datetime object
-        df.loc[index, "date"].update(
-            df.loc[index, "date"].apply(
-                lambda x: datetime.datetime.strptime(
-                    x, "%a, %d %b %Y %H:%M:%S %z"
-                )
+    )
+    return df
+
+
+def mlist_to_mbox(msgs: MailList, dir_out: str, filename: str) -> None:
+    """
+    Saves a List[mailbox.mboxMessage] as .mbox file.
+    For a clearer definition on what a mailing list is, see:
+    bigbang.ingress.abstract.AbstractList
+    """
+    # create filepath
+    filepath = f"{dir_out}/{filename}.mbox"
+    # delete file if there is one at the filepath
+    if Path(filepath).is_file():
+        Path(filepath).unlink()
+    mbox = mailbox.mbox(filepath)
+    mbox.lock()
+    for msg in msgs:
+        try:
+            mbox.add(msg)
+        except Exception as e:
+            logger.info(
+                f'Add to .mbox error for {msg["Archived-At"]} because, {e}'
             )
+    mbox.flush()
+    mbox.unlock()
+    logger.info(f"The list {filename} is saved at {filepath}.")
+    mbox.lock()
+    mbox.add(msg)
+    mbox.flush()
+    mbox.unlock()
+
+
+def mlistdom_to_dict(
+    mlists: MailListDomain, include_body: bool = True
+) -> Dict[str, List[str]]:
+    """
+    Handles data type transformation from a List[AbstractList] to a
+    `Dictionary`.
+    For a clearer definition on what a mailing archive is, see:
+    bigbang.ingress.abstract.AbstractArchive
+    """
+    nr_msgs = 0
+    for ii, mlist in enumerate(mlists):
+        dic_mlist = mlist_to_dict(mlist.messages, include_body)
+        if ii == 0:
+            dic_mlistdom = dic_mlist
+            dic_mlistdom["mailing-list"] = [mlist.name] * len(mlist)
+        else:
+            # add mlist items to mlistdom
+            for key, value in dic_mlist.items():
+                if key not in dic_mlistdom.keys():
+                    dic_mlistdom[key] = [np.nan] * nr_msgs
+                dic_mlistdom[key].extend(value)
+            # if mlist does not contain items that are in mlistdom
+            key_miss = list(set(dic_mlistdom.keys()) - set(dic_mlist.keys()))
+            key_miss.remove("mailing-list")
+            for key in key_miss:
+                dic_mlistdom[key].extend([np.nan] * len(mlist))
+
+            dic_mlistdom["mailing-list"].extend([mlist.name] * len(mlist))
+        nr_msgs += len(mlist)
+    lengths = [len(value) for value in dic_mlistdom.values()]
+    assert all([diff == 0 for diff in np.diff(lengths)])
+    return dic_mlistdom
+
+
+def mlistdom_to_pandas_dataframe(
+    mlists: MailListDomain, include_body: bool = True
+) -> pd.DataFrame:
+    """
+    Handles data type transformation from a List[AbstractList] to a
+    pandas.DataFrame.
+    For a clearer definition on what a mailing archive is, see:
+    bigbang.ingress.abstract.AbstractArchive
+    """
+    df = pd.DataFrame(mlistdom_to_dict(mlists, include_body)).set_index(
+        "message-id"
+    )
+    # get index of date-times
+    index = np.array(
+        [
+            True
+            if (isinstance(dt, str)) and (len(dt) > 10) and not pd.isna(dt)
+            else False
+            for i, dt in enumerate(df["date"])
+        ],
+        dtype="bool",
+    )
+    # convert data type from string to datetime.datetime object
+    df.loc[index, "date"].update(
+        df.loc[index, "date"].apply(
+            lambda x: datetime.datetime.strptime(x, "%a, %d %b %Y %H:%M:%S %z")
         )
-        return df
-
-    def to_mbox(mlists: list, dir_out: str):
-        """
-        Save Archive content to .mbox files
-        """
-        for mlist in mlists:
-            mlist.to_mbox(dir_out)
+    )
+    return df
+
+
+def mlistdom_to_mbox(mlists: MailListDomain, dir_out: str):
+    """
+    Saves a List[AbstractList] as .mbox file.
+    For a clearer definition on what a mailing archive is, see:
+    bigbang.ingress.abstract.AbstractArchive
+    """
+    for mlist in mlists:
+        mlist.to_mbox(dir_out)
 
 
 def get_paths_to_files_in_directory(
diff --git a/bigbang/data_types.py b/bigbang/data_types.py
new file mode 100644
index 00000000..86184754
--- /dev/null
+++ b/bigbang/data_types.py
@@ -0,0 +1,6 @@
+from mailbox import mboxMessage
+from typing import List
+
+Message = mboxMessage
+MailList = List[Message]
+MailListDomain = List[MailList]
diff --git a/bigbang/ingress/__init__.py b/bigbang/ingress/__init__.py
new file mode 100644
index 00000000..d925fd8d
--- /dev/null
+++ b/bigbang/ingress/__init__.py
@@ -0,0 +1,15 @@
+from bigbang.ingress.abstract import (
+    AbstractMessageParser,
+    AbstractMailList,
+    AbstractMailListDomain,
+)
+from bigbang.ingress.listserv import (
+    ListservMessageParser,
+    ListservMailList,
+    ListservMailListDomain,
+)
+from bigbang.ingress.w3c import (
+    W3CMessageParser,
+    W3CMailList,
+    W3CMailListDomain,
+)
diff --git a/bigbang/ingress/abstract.py b/bigbang/ingress/abstract.py
new file mode 100644
index 00000000..fe300dd3
--- /dev/null
+++ b/bigbang/ingress/abstract.py
@@ -0,0 +1,649 @@
+from tqdm import tqdm
+import time
+import logging
+import os
+import email
+from abc import ABC, abstractmethod
+import mailbox
+from mailbox import mboxMessage
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import pandas as pd
+import requests
+import yaml
+from bs4 import BeautifulSoup
+
+from config.config import CONFIG
+from bigbang.utils import get_paths_to_files_in_directory
+import bigbang.bigbang_io as bio
+from bigbang.data_types import Message, MailList
+from bigbang.ingress.utils import (
+    get_website_content,
+    get_auth_session,
+    set_website_preference_for_header,
+)
+
+filepath_auth = CONFIG.config_path + "authentication.yaml"
+directory_project = str(Path(os.path.abspath(__file__)).parent.parent)
+logging.basicConfig(
+    filename=directory_project + "/abstract.scraping.log",
+    filemode="w",
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+class AbstractMessageParserWarning(BaseException):
+    """Base class for AbstractMessageParser class specific exceptions"""
+
+    pass
+
+
+class AbstractMailListWarning(BaseException):
+    """Base class for AbstractMailList class specific exceptions"""
+
+    pass
+
+
+class AbstractMailListDomainWarning(BaseException):
+    """Base class for AbstractMailListDomain class specific exceptions"""
+
+    pass
+
+
+class AbstractMessageParser(ABC):
+    """
+    This class handles the creation of an mailbox.mboxMessage object
+    (using the from_*() methods) and its storage in various other file formats
+    (using the to_*() methods) that can be saved on the local memory.
+    """
+
+    def __init__(
+        self,
+        website=False,
+        url_login: Optional[str] = None,
+        url_pref: Optional[str] = None,
+        login: Optional[Dict[str, str]] = {"username": None, "password": None},
+        session: Optional[requests.Session] = None,
+    ):
+        if website:
+            if (session is None) and (url_login is not None):
+                session = get_auth_session(url_login, **login)
+                session = set_website_preference_for_header(url_pref, session)
+            self.session = session
+
+    def create_email_message(
+        self,
+        archived_at: str,
+        body: str,
+        **header,
+    ) -> Message:
+        """
+        Parameters
+        ----------
+        archived_at : URL to the Email message.
+        body : String that contains the body of the message.
+        header : Dictionary that contains all available header fields of the
+            message.
+        """
+        msg = email.message.EmailMessage()
+        if body is not None:
+            try:
+                msg.set_content(body)  # don't use charset="utf-16"
+            except Exception:
+                # UnicodeEncodeError: 'utf-16' codec can't encode character
+                # '\ud83d' in position 8638: surrogates not allowed
+                pass
+        for key, value in header.items():
+            if "content-type" == key:
+                msg.set_param("Content-Type", value)
+            elif "mime-version" == key:
+                msg.set_param("MIME-Version", value)
+            elif "content-transfer-encoding" == key:
+                msg.set_param("Content-Transfer-Encoding", value)
+            else:
+                try:
+                    # TODO: find out why it sometimes raises
+                    # email/_header_value_parser.py
+                    # IndexError: list index out of range.
+                    # Also look into UTF-8 encoding.
+                    msg[key] = value
+                except Exception:
+                    pass
+        if (
+            (msg["Message-ID"] is None)
+            and (msg["Date"] is not None)
+            and (msg["From"] is not None)
+        ):
+            msg["Message-ID"] = archived_at.split("/")[-1]
+        # convert to `EmailMessage` to `mboxMessage`
+        mbox_msg = mboxMessage(msg)
+        mbox_msg.add_header("Archived-At", "<" + archived_at + ">")
+        return mbox_msg
+
+    def from_url(
+        self,
+        list_name: str,
+        url: str,
+        fields: str = "total",
+    ) -> Message:
+        """
+        Parameters
+        ----------
+        list_name : The name of the mailing list.
+        url : URL of this Email
+        fields : Indicates whether to return 'header', 'body' or 'total'/both or
+            the Email. The latter is the default.
+        """
+        soup = get_website_content(url, session=self.session)
+        if soup == "RequestException":
+            body = "RequestException"
+            header = self.empty_header
+        else:
+            if fields in ["header", "total"]:
+                header = self._get_header_from_html(soup)
+            else:
+                header = self.empty_header
+            if fields in ["body", "total"]:
+                body = self._get_body_from_html(list_name, url, soup)
+            else:
+                body = None
+        return self.create_email_message(url, body, **header)
+
+    @abstractmethod
+    def _get_header_from_html(self, soup: BeautifulSoup) -> Dict[str, str]:
+        pass
+
+    @abstractmethod
+    def _get_body_from_html(
+        self, list_name: str, url: str, soup: BeautifulSoup
+    ) -> Union[str, None]:
+        pass
+
+    @staticmethod
+    def to_dict(msg: Message) -> Dict[str, List[str]]:
+        """Convert mboxMessage to a Dictionary"""
+        return bio.email_to_dict(msg)
+
+    @staticmethod
+    def to_pandas_dataframe(msg: Message) -> pd.DataFrame:
+        """Convert mboxMessage to a pandas.DataFrame"""
+        return bio.email_to_pandas_dataframe(msg)
+
+    @staticmethod
+    def to_mbox(msg: Message, filepath: str):
+        """
+        Parameters
+        ----------
+        msg : The Email.
+        filepath : Path to file in which the Email will be stored.
+        """
+        return bio.email_to_mbox(msg, filepath)
+
+
+class AbstractMailList(ABC):
+    """
+    This class handles the scraping of a all public Emails contained in a single
+    mailing list. To be more precise, each contributor to a mailing list sends
+    their message to an Email address that has the following structure:
+    <mailing_list_name>@<mail_list_domain_name>.
+    Thus, this class contains all Emails send to a specific <mailing_list_name>
+    (the Email localpart).
+
+    Parameters
+    ----------
+    name : The of whom the list (e.g. 3GPP_COMMON_IMS_XFER, IEEESCO-DIFUSION, ...)
+    source : Contains the information of the location of the mailing list.
+        It can be either an URL where the list or a path to the file(s).
+    msgs : List of mboxMessage objects
+
+    Methods
+    -------
+    from_url()
+    from_messages()
+    from_mbox()
+    get_message_urls()
+    get_messages_from_url()
+    get_index_of_elements_in_selection()
+    get_name_from_url()
+    to_dict()
+    to_pandas_dataframe()
+    to_mbox()
+    """
+
+    def __init__(
+        self,
+        name: str,
+        source: Union[List[str], str],
+        msgs: MailList,
+    ):
+        self.name = name
+        self.source = source
+        self.messages = msgs
+
+    def __len__(self) -> int:
+        """Get number of messsages within the mailing list."""
+        return len(self.messages)
+
+    def __iter__(self):
+        """Iterate over each message within the mailing list."""
+        return iter(self.messages)
+
+    def __getitem__(self, index) -> Message:
+        """Get specific message at position `index` within the mailing list."""
+        return self.messages[index]
+
+    @classmethod
+    @abstractmethod
+    def from_url(
+        cls,
+        name: str,
+        url: str,
+        select: Optional[dict] = {"fields": "total"},
+        url_login: Optional[str] = None,
+        url_pref: Optional[str] = None,
+        login: Optional[Dict[str, str]] = {"username": None, "password": None},
+        session: Optional[requests.Session] = None,
+    ) -> "AbstractMailList":
+        """
+        Parameters
+        ----------
+        name : Name of the mailing list.
+        url : URL to the mailing list.
+        select : Selection criteria that can filter messages by:
+            - content, i.e. header and/or body
+            - period, i.e. written in a certain year, month, week-of-month
+        url_login : URL to the 'Log In' page
+        url_pref : URL to the 'Preferences'/settings page
+        login : Login credentials (username and password) that were used to set
+            up AuthSession.
+        session : requests.Session() object for the Email list domain website.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def from_messages(
+        cls,
+        name: str,
+        url: str,
+        messages: Union[List[str], MailList],
+        fields: str = "total",
+        url_login: str = None,
+        url_pref: str = None,
+        login: Optional[Dict[str, str]] = {"username": None, "password": None},
+        session: Optional[str] = None,
+    ) -> "AbstractMailList":
+        """
+        Parameters
+        ----------
+        name : Name of the list of messages, e.g. 'public-bigdata'
+        url : URL to the Email list.
+        messages : Can either be a list of URLs to specific messages
+            or a list of `mboxMessage` objects.
+        url_login : URL to the 'Log In' page.
+        url_pref : URL to the 'Preferences'/settings page.
+        login : Login credentials (username and password) that were used to set
+            up AuthSession.
+        session : requests.Session() object for the Email list domain website.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def from_mbox(cls, name: str, filepath: str) -> "AbstractMailList":
+        """
+        Parameters
+        ----------
+        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'.
+        filepath : Path to file in which mailing list is stored.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def get_message_urls(
+        cls,
+        name: str,
+        url: str,
+        select: Optional[dict] = None,
+    ) -> List[str]:
+        """
+        Parameters
+        ----------
+        name : Name of the list of messages, e.g. 'public-bigdata'
+        url : URL to the mailing list.
+        select : Selection criteria that can filter messages by:
+            - content, i.e. header and/or body
+            - period, i.e. written in a certain year and month
+
+        Returns
+        -------
+        List of all selected URLs of the messages in the mailing list.
+        """
+        pass
+
+    @staticmethod
+    def get_messages_from_urls(
+        name: str,
+        msg_urls: list,
+        msg_parser,
+        fields: Optional[str] = "total",
+    ) -> MailList:
+        """
+        Generator that returns all messages within a certain period
+        (e.g. January 2021, Week 5).
+
+        Parameters
+        ----------
+        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'
+        url : URL to the mailing list.
+        fields : Content, i.e. header and/or body
+        """
+        # get all message contents
+        msgs = []
+        for msg_url in tqdm(msg_urls, ascii=True, desc=name):
+            msg = msg_parser.from_url(
+                list_name=name,
+                url=msg_url,
+                fields=fields,
+            )
+            if msg.get_payload() == "RequestException":
+                time.sleep(30)
+            else:
+                msgs.append(msg)
+                logger.info(f"Recorded the message {msg_url}.")
+            # wait between loading messages, for politeness
+            time.sleep(1)
+        return msgs
+
+    @staticmethod
+    def get_index_of_elements_in_selection(
+        times: List[Union[int, str]],
+        urls: List[str],
+        filtr: Union[tuple, list, int, str],
+    ) -> List[int]:
+        """
+        Filter out messages that where in a specific period. Period here is a set
+        containing units of year, month, and week-of-month which can have the following
+        example elements:
+            - years: (1992, 2010), [2000, 2008], 2021
+            - months: ["January", "July"], "November"
+            - weeks: (1, 4), [1, 5], 2
+
+        Parameters
+        ----------
+        times : A list containing information of the period for each
+            group of mboxMessage.
+        urls : Corresponding URLs of each group of mboxMessage of which the
+            period info is contained in `times`.
+        filtr : Containing info on what should be filtered.
+
+        Returns
+        -------
+        Indices of to the elements in `times`/`ursl`.
+        """
+        if isinstance(filtr, tuple):
+            # filter year or week in range
+            cond = lambda x: (np.min(filtr) <= x <= np.max(filtr))
+        if isinstance(filtr, list):
+            # filter in year, week, or month in list
+            cond = lambda x: x in filtr
+        if isinstance(filtr, int):
+            # filter specific year or week
+            cond = lambda x: x == filtr
+        if isinstance(filtr, str):
+            # filter specific month
+            cond = lambda x: x == filtr
+        return [idx for idx, time in enumerate(times) if cond(time)]
+
+    @abstractmethod
+    def get_name_from_url(url: str) -> str:
+        pass
+
+    def to_dict(self, include_body: bool = True) -> Dict[str, List[str]]:
+        """
+        Parameters
+        ----------
+        include_body : A boolean that indicates whether the message body should
+            be included or not.
+
+        Returns
+        -------
+        A Dictionary with the first key layer being the header field names and
+        the "body" key. Each value field is a list containing the respective
+        header field contents arranged by the order as they were scraped from
+        the web. This format makes the conversion to a pandas.DataFrame easier.
+        """
+        return bio.mlist_to_dict(self.messages, include_body)
+
+    def to_pandas_dataframe(self, include_body: bool = True) -> pd.DataFrame:
+        """
+        Parameters
+        ----------
+        include_body : A boolean that indicates whether the message body should
+            be included or not.
+
+        Returns
+        -------
+        Converts the mailing list into a pandas.DataFrame object in which each
+        row represents an Email.
+        """
+        return bio.mlist_to_pandas_dataframe(self.messages, include_body)
+
+    def to_mbox(self, dir_out: str, filename: Optional[str] = None):
+        """Safe mailing list to .mbox files."""
+        if filename is None:
+            bio.mlist_to_mbox(self.messages, dir_out, self.name)
+        else:
+            bio.mlist_to_mbox(self.messages, dir_out, filename)
+
+
+class AbstractMailListDomain(ABC):
+    """
+    This class handles the scraping of a all public Emails contained in a mail
+    list domain. To be more precise, each contributor to a mailing archive sends
+    their message to an Email address that has the following structure:
+    <mailing_list_name>@<mail_list_domain_name>.
+    Thus, this class contains all Emails send to <mail_list_domain_name>
+    (the Email domain name). These Emails are contained in a list of
+    `AbstractMailList` types, such that it is known to which <mailing_list_name>
+    (the Email localpart) was send.
+
+    Parameters
+    ----------
+    name : The mail list domain name (e.g. 3GPP, IEEE, W3C)
+    url : The URL where the archive lives
+    lists : A list containing the mailing lists as `AbstractMailList` types
+
+    Methods
+    -------
+    from_url()
+    from_mailing_lists()
+    from_mbox()
+    get_lists_from_url()
+    to_dict()
+    to_pandas_dataframe()
+    to_mbox()
+    """
+
+    def __init__(
+        self, name: str, url: str, lists: List[Union[AbstractMailList, str]]
+    ):
+        self.name = name
+        self.url = url
+        self.lists = lists
+
+    def __len__(self):
+        """Get number of mailing lists within the mail list domain."""
+        return len(self.lists)
+
+    def __iter__(self):
+        """Iterate over each mailing list within the mail list domain."""
+        return iter(self.lists)
+
+    def __getitem__(self, index):
+        """Get specific mailing list at position `index` from the mail list domain."""
+        return self.lists[index]
+
+    @classmethod
+    @abstractmethod
+    def from_url(
+        cls,
+        name: str,
+        url_root: str,
+        url_home: Optional[str] = None,
+        select: Optional[dict] = None,
+        url_login: Optional[str] = None,
+        url_pref: Optional[str] = None,
+        login: Optional[Dict[str, str]] = {"username": None, "password": None},
+        session: Optional[str] = None,
+        instant_save: bool = True,
+        only_mlist_urls: bool = True,
+    ) -> "AbstractMailListDomain":
+        """
+        Create a mail list domain from a given URL.
+        Parameters
+        ----------
+        name : Email list domain name, such that multiple instances of
+            `AbstractMailListDomain` can easily be distinguished.
+        url_root : The invariant root URL that does not change no matter what
+            part of the mail list domain we access.
+        url_home : The 'home' space of the mail list domain. This is required as
+            it contains the different sections which we obtain using `get_sections()`.
+        select: Selection criteria that can filter messages by:
+            - content, i.e. header and/or body
+            - period, i.e. written in a certain year, month, week-of-month
+        url_login : URL to the 'Log In' page.
+        url_pref : URL to the 'Preferences'/settings page.
+        login : Login credentials (username and password) that were used to set
+            up AuthSession.
+        session : requests.Session() object for the mail list domain website.
+        instant_save : Boolean giving the choice to save a `AbstractMailList` as
+            soon as it is completely scraped or collect entire mail list domain. The
+            prior is recommended if a large number of mailing lists are
+            scraped which can require a lot of memory and time.
+        only_list_urls : Boolean giving the choice to collect only `AbstractMailList`
+            URLs or also their contents.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def from_mailing_lists(
+        cls,
+        name: str,
+        url_root: str,
+        url_mailing_lists: Union[List[str], List[AbstractMailList]],
+        select: Optional[dict] = None,
+        url_login: Optional[str] = None,
+        url_pref: Optional[str] = None,
+        login: Optional[Dict[str, str]] = {"username": None, "password": None},
+        session: Optional[str] = None,
+        only_mlist_urls: bool = True,
+        instant_save: Optional[bool] = True,
+    ) -> "AbstractMailListDomain":
+        """
+        Create mailing mail list domain from a given list of 'AbstractMailList` instances or URLs
+        pointing to mailing lists.
+
+        Parameters
+        ----------
+        name : mail list domain name, such that multiple instances of
+            `AbstractMailListDomain` can easily be distinguished.
+        url_root : The invariant root URL that does not change no matter what
+            part of the mail list domain we access.
+        url_mailing_lists : This argument can either be a list of `AbstractMailList`
+            objects or a list of string containing the URLs to the mailing
+            list of interest.
+        url_login : URL to the 'Log In' page.
+        url_pref : URL to the 'Preferences'/settings page.
+        login : Login credentials (username and password) that were used to set
+            up AuthSession.
+        session : requests.Session() object for the mail list domain website.
+        only_list_urls : Boolean giving the choice to collect only mailing list
+            URLs or also their contents.
+        instant_save : Boolean giving the choice to save a `AbstractMailList` as
+            soon as it is completely scraped or collect entire mail list domain. The
+            prior is recommended if a large number of mailing lists are
+            scraped which can require a lot of memory and time.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def from_mbox(
+        cls,
+        name: str,
+        directorypath: str,
+        filedsc: str = "*.mbox",
+    ) -> "AbstractMailListDomain":
+        """
+        Parameters
+        ----------
+        name : mail list domain name, such that multiple instances of
+            `AbstractMailListDomain` can easily be distinguished.
+        directorypath : Path to the folder in which `AbstractMailListDomain` is stored.
+        filedsc : Optional filter that only reads files matching the description.
+            By default all files with an mbox extension are read.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def get_lists_from_url(
+        url_root: str,
+        url_home: str,
+        select: dict,
+        session: Optional[str] = None,
+        instant_save: bool = True,
+        only_mlist_urls: bool = True,
+    ) -> List[Union[AbstractMailList, str]]:
+        """
+        Created dictionary of all lists in the mail list domain.
+
+        Parameters
+        ----------
+        url_root : The invariant root URL that does not change no matter what
+            part of the mail list domain we access.
+        url_home : The 'home' space of the mail list domain. This is required as
+            it contains the different sections which we obtain using `get_sections()`.
+        select : Selection criteria that can filter messages by:
+            - content, i.e. header and/or body
+            - period, i.e. written in a certain year, month, week-of-month
+        session : requests.Session() object for the mail list domain website.
+        instant_save : Boolean giving the choice to save a `AbstractMailList` as
+            soon as it is completely scraped or collect entire mail list domain. The
+            prior is recommended if a large number of mailing lists are
+            scraped which can require a lot of memory and time.
+        only_list_urls : Boolean giving the choice to collect only `AbstractMailList`
+            URLs or also their contents.
+
+        Returns
+        -------
+        archive_dict : the keys are the names of the lists and the value their url
+        """
+        pass
+
+    def to_dict(self, include_body: bool = True) -> Dict[str, List[str]]:
+        """
+        Concatenates mailing list dictionaries created using
+        `AbstractMailList.to_dict()`.
+        """
+        return bio.mlistdom_to_dict(self.lists, include_body)
+
+    def to_pandas_dataframe(self, include_body: bool = True) -> pd.DataFrame:
+        """
+        Concatenates mailing list pandas.DataFrames created using
+        `AbstractMailList.to_pandas_dataframe()`.
+        """
+        return bio.mlistdom_to_pandas_dataframe(self.lists, include_body)
+
+    def to_mbox(self, dir_out: str):
+        """
+        Save mail list domain content to .mbox files
+        """
+        bio.mlistdom_to_mbox(self.lists, dir_out)
diff --git a/bigbang/listserv.py b/bigbang/ingress/listserv.py
similarity index 50%
rename from bigbang/listserv.py
rename to bigbang/ingress/listserv.py
index f79a32b4..3a40ea30 100644
--- a/bigbang/listserv.py
+++ b/bigbang/ingress/listserv.py
@@ -1,4 +1,3 @@
-import datetime
 import email
 import logging
 import os
@@ -17,14 +16,20 @@
 import requests
 import yaml
 from bs4 import BeautifulSoup
-from tqdm import tqdm
 
 from config.config import CONFIG
 
-from bigbang.bigbang_io import (
-    ListservMessageIO,
-    ListservListIO,
-    ListservArchiveIO,
+import bigbang.bigbang_io as bio
+from bigbang.data_types import MailList
+from bigbang.ingress import (
+    AbstractMessageParser,
+    AbstractMailList,
+    AbstractMailListDomain,
+)
+from bigbang.ingress.utils import (
+    get_website_content,
+    set_website_preference_for_header,
+    get_auth_session,
 )
 from bigbang.utils import (
     get_paths_to_files_in_directory,
@@ -43,24 +48,24 @@
 
 
 class ListservMessageParserWarning(BaseException):
-    """Base class for Archive class specific exceptions"""
+    """Base class for ListservMessageParser class specific exceptions"""
 
     pass
 
 
-class ListservListWarning(BaseException):
-    """Base class for Archive class specific exceptions"""
+class ListservMailListWarning(BaseException):
+    """Base class for ListservMailList class specific exceptions"""
 
     pass
 
 
-class ListservArchiveWarning(BaseException):
-    """Base class for Archive class specific exceptions"""
+class ListservMailListDomainWarning(BaseException):
+    """Base class for ListservMailListDomain class specific exceptions"""
 
     pass
 
 
-class ListservMessageParser(email.parser.Parser):
+class ListservMessageParser(AbstractMessageParser, email.parser.Parser):
     """
     This class handles the creation of an mailbox.mboxMessage object
     (using the from_*() methods) and its storage in various other file formats
@@ -73,8 +78,8 @@ class ListservMessageParser(email.parser.Parser):
     url_login : URL to the 'Log In' page.
     url_pref : URL to the 'Preferences'/settings page.
     login : Login credentials (username and password) that were used to set
-        up AuthSession. You can create your own for the 3GPP archive.
-    session : requests.Session() object for the Email archive website.
+        up AuthSession. You can create your own for the 3GPP mail list domain.
+    session : requests.Session() object for the mail list domain website.
 
     Methods
     -------
@@ -84,7 +89,6 @@ class ListservMessageParser(email.parser.Parser):
     _get_body_from_html()
     _get_header_from_listserv_file()
     _get_body_from_listserv_file()
-    get_datetime()
 
     Example
     -------
@@ -94,8 +98,8 @@ class ListservMessageParser(email.parser.Parser):
     >>>     login={"username": <your_username>, "password": <your_password>},
     >>> )
 
-    To obtain the Email message content and return it as `email.message.EmailMessage`
-    object, you need to do the following:
+    To obtain the Email message content and return it as `mboxMessage` object,
+    you need to do the following:
     >>> msg = msg_parser.from_url(
     >>>     list_name="3GPP_TSG_RAN_DRAFTS",
     >>>     url="https://list.etsi.org/scripts/wa.exe?A2=ind2010B&L=3GPP_TSG_RAN_DRAFTS&O=D&P=29883",
@@ -105,99 +109,6 @@ class ListservMessageParser(email.parser.Parser):
 
     empty_header = {}
 
-    def __init__(
-        self,
-        website=False,
-        url_login: str = "https://list.etsi.org/scripts/wa.exe?LOGON",
-        url_pref: str = "https://list.etsi.org/scripts/wa.exe?PREF",
-        login: Optional[Dict[str, str]] = {"username": None, "password": None},
-        session: Optional[requests.Session] = None,
-    ):
-        if website:
-            if session is None:
-                session = get_auth_session(url_login, **login)
-                session = set_website_preference_for_header(url_pref, session)
-            self.session = session
-
-    def create_email_message(
-        self,
-        archived_at: str,
-        body: str,
-        **header,
-    ) -> mboxMessage:
-        """
-        Parameters
-        ----------
-        archived_at : URL to the Email message.
-        body : String that contains the body of the message.
-        header : Dictionary that contains all available header fields of the
-            message.
-        """
-        # crea EmailMessage
-        msg = email.message.EmailMessage()
-        if body is not None:
-            try:
-                msg.set_content(body)  # don't use charset="utf-16"
-            except Exception:
-                # UnicodeEncodeError: 'utf-16' codec can't encode character
-                # '\ud83d' in position 8638: surrogates not allowed
-                pass
-        for key, value in header.items():
-            if "content-type" == key:
-                msg.set_param("Content-Type", value)
-            elif "mime-version" == key:
-                msg.set_param("MIME-Version", value)
-            elif "content-transfer-encoding" == key:
-                msg.set_param("Content-Transfer-Encoding", value)
-            else:
-                try:
-                    # TODO: find out why it sometimes raises
-                    # email/_header_value_parser.py
-                    # IndexError: list index out of range.
-                    # Also look into UTF-8 encoding.
-                    msg[key] = value
-                except Exception:
-                    pass
-        if (
-            (msg["Message-ID"] is None)
-            and (msg["Date"] is not None)
-            and (msg["From"] is not None)
-        ):
-            msg["Message-ID"] = archived_at.split("/")[-1]
-        # convert to EmailMessage to mboxMessage
-        mbox_msg = mboxMessage(msg)
-        mbox_msg.add_header("Archived-At", "<" + archived_at + ">")
-        return mbox_msg
-
-    def from_url(
-        self,
-        list_name: str,
-        url: str,
-        fields: str = "total",
-    ) -> mboxMessage:
-        """
-        Parameters
-        ----------
-        list_name : The name of the LISTSERV Email list.
-        url : URL of this Email
-        fields : Indicates whether to return 'header', 'body' or 'total'/both or
-            the Email. The latter is the default.
-        """
-        soup = get_website_content(url, session=self.session)
-        if soup == "RequestException":
-            body = "RequestException"
-            header = self.empty_header
-        else:
-            if fields in ["header", "total"]:
-                header = self._get_header_from_html(soup)
-            else:
-                header = self.empty_header
-            if fields in ["body", "total"]:
-                body = self._get_body_from_html(list_name, url, soup)
-            else:
-                body = None
-        return self.create_email_message(url, body, **header)
-
     def from_listserv_file(
         self,
         list_name: str,
@@ -432,58 +343,16 @@ def _get_body_from_html(
             )
             return None
 
-    @staticmethod
-    def get_datetime(line: str) -> str:
-        """
-        Parameters
-        ----------
-        line : String that contains date and time.
-        """
-        line = (" ").join(line.split(" ")[:-1]).lstrip()
-        # convert format to local version of date and time
-        date_time_obj = datetime.datetime.strptime(
-            line, "%a, %d %b %Y %H:%M:%S"
-        )
-        return date_time_obj.strftime("%c")
-
-    @staticmethod
-    def create_message_id(date: str, from_address: str) -> str:
-        """
-        Parameters
-        ----------
-        date : Date and time of Email.
-        from_address : The sender address of the Email.
-        """
-        message_id = (".").join([date, from_address])
-        # remove special characters
-        message_id = re.sub(r"[^a-zA-Z0-9]+", "", message_id)
-        return message_id
-
-    @staticmethod
-    def to_dict(msg: mboxMessage) -> Dict[str, List[str]]:
-        """Convert mboxMessage to a Dictionary"""
-        return ListservMessageIO.to_dict(msg)
-
-    @staticmethod
-    def to_pandas_dataframe(msg: mboxMessage) -> pd.DataFrame:
-        """Convert mboxMessage to a pandas.DataFrame"""
-        return ListservMessageIO.to_pandas_dataframe(msg)
 
-    @staticmethod
-    def to_mbox(msg: mboxMessage, filepath: str):
-        """
-        Parameters
-        ----------
-        msg : The Email.
-        filepath : Path to file in which the Email will be stored.
-        """
-        return ListservMessageIO.to_mbox(msg, filepath)
-
-
-class ListservList(ListservListIO):
+class ListservMailList(AbstractMailList):
     """
-    This class handles the scraping of a single mailing list of a public archive
-    in the LISTSERV 16.5 format.
+    This class handles the scraping of a all public Emails contained in a single
+    mailing list in the LISTSERV 16.5 and 17 format.
+    To be more precise, each contributor to a mailing list sends
+    their message to an Email address that has the following structure:
+    <mailing_list_name>@LIST.ETSI.ORG.
+    Thus, this class contains all Emails send to a specific <mailing_list_name>
+    (the Email localpart, such as "3GPP_TSG_CT_WG1" or "3GPP_TSG_CT_WG3_108E_MAIN").
 
     Parameters
     ----------
@@ -494,26 +363,19 @@ class ListservList(ListservListIO):
 
     Methods
     -------
-    from_url()
-    from_messages()
-    from_mbox()
+    All methods in the `AbstractMailList` class in addition to:
     from_listserv_files()
     from_listserv_directories()
-    get_messages_from_url()
-    get_message_urls()
     get_period_urls()
     get_line_numbers_of_header_starts()
     get_index_of_elements_in_selection()
-    to_dict()
-    to_pandas_dataframe()
-    to_mbox()
 
 
     Example
     -------
     To scrape a Listserv mailing list from an URL and store it in
     run-time memory, we do the following
-    >>> mlist = ListservList.from_url(
+    >>> mlist = ListservMailList.from_url(
     >>>     name="IEEE-TEST",
     >>>     url="https://listserv.ieee.org/cgi-bin/wa?A0=IEEE-TEST",
     >>>     select={
@@ -529,124 +391,71 @@ class ListservList(ListservListIO):
     >>> mlist.to_mbox(path_to_file)
     """
 
-    def __init__(
-        self,
-        name: str,
-        source: Union[List[str], str],
-        msgs: List[mboxMessage],
-    ):
-        self.name = name
-        self.source = source
-        self.messages = msgs
-
-    def __len__(self) -> int:
-        """Get number of messsages within the mailing list."""
-        return len(self.messages)
-
-    def __iter__(self):
-        """Iterate over each message within the mailing list."""
-        return iter(self.messages)
-
-    def __getitem__(self, index) -> mboxMessage:
-        """Get specific message at position `index` within the mailing list."""
-        return self.messages[index]
-
     @classmethod
     def from_url(
         cls,
         name: str,
         url: str,
-        select: Optional[dict] = None,
+        select: Optional[dict] = {"fields": "total"},
         url_login: str = "https://list.etsi.org/scripts/wa.exe?LOGON",
         url_pref: str = "https://list.etsi.org/scripts/wa.exe?PREF",
         login: Optional[Dict[str, str]] = {"username": None, "password": None},
         session: Optional[requests.Session] = None,
-    ) -> "ListservList":
-        """
-        Parameters
-        ----------
-        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'
-        url : URL to the LISTSERV list.
-        select : Selection criteria that can filter messages by:
-            - content, i.e. header and/or body
-            - period, i.e. written in a certain year, month, week-of-month
-        url_login : URL to the 'Log In' page
-        url_pref : URL to the 'Preferences'/settings page
-        login : Login credentials (username and password) that were used to set
-            up AuthSession. You can create your own for the 3GPP archive.
-        session : requests.Session() object for the Email archive website.
-        """
-        if session is None:
+    ) -> "ListservMailList":
+        """Docstring in `AbstractMailList`."""
+        if (session is None) and (url_login is not None):
             session = get_auth_session(url_login, **login)
             session = set_website_preference_for_header(url_pref, session)
-        if select is None:
-            select = {"fields": "total"}
-        elif "fields" not in list(select.keys()):
+        if "fields" not in list(select.keys()):
             select["fields"] = "total"
-        msgs = cls.get_messages_from_url(name, url, select, session)
-        return cls.from_messages(name, url, msgs)
+        msg_urls = cls.get_message_urls(name, url, select)
+        return cls.from_messages(
+            name,
+            url,
+            msg_urls,
+            select["fields"],
+            url_login,
+            url_pref,
+            login,
+            session,
+        )
 
     @classmethod
     def from_messages(
         cls,
         name: str,
         url: str,
-        messages: List[Union[str, mboxMessage]],
+        messages: Union[List[str], MailList],
         fields: str = "total",
         url_login: str = "https://list.etsi.org/scripts/wa.exe?LOGON",
         url_pref: str = "https://list.etsi.org/scripts/wa.exe?PREF",
         login: Optional[Dict[str, str]] = {"username": None, "password": None},
         session: Optional[str] = None,
-    ) -> "ListservList":
-        """
-        Parameters
-        ----------
-        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'
-        url : URL to the LISTSERV Email list.
-        messages : Can either be a list of URLs to specific LISTSERV messages
-            or a list of `mboxMessage` objects.
-        url_login : URL to the 'Log In' page.
-        url_pref : URL to the 'Preferences'/settings page.
-        login : Login credentials (username and password) that were used to set
-            up AuthSession. You can create your own for the 3GPP archive.
-        session : requests.Session() object for the LISTSERV Email archive website.
-        """
+    ) -> "ListservMailList":
+        """Docstring in `AbstractMailList`."""
         if not messages:
-            # create empty ListservList for ListservArchive
-            msgs = messages
-        elif isinstance(messages[0], str):
-            # create ListservList from message URLs
             msgs = []
+        elif isinstance(messages[0], str):
+            if (session is None) and (url_login is not None):
+                session = get_auth_session(url_login, **login)
+                session = set_website_preference_for_header(url_pref, session)
             msg_parser = ListservMessageParser(
                 website=True,
                 url_login=url_login,
                 login=login,
+                session=session,
+            )
+            msgs = super().get_messages_from_urls(
+                name, messages, msg_parser, fields
             )
-            for msg_url in tqdm(messages, ascii=True, desc=name):
-                msg = msg_parser.from_url(
-                    list_name=name,
-                    url=msg_url,
-                    fields=fields,
-                )
-                if msg.get_payload() == "RequestException":
-                    time.sleep(30)
-                else:
-                    msgs.append(msg)
-                    logger.info(f"Recorded the message {msg_url}.")
         else:
-            # create ListservList from list of mboxMessage
             msgs = messages
         return cls(name, url, msgs)
 
     @classmethod
-    def from_mbox(cls, name: str, filepath: str) -> "ListservList":
-        """
-        Parameters
-        ----------
-        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'.
-        filepath : Path to file in which mailing list is stored.
-        """
-        msgs = ListservListIO.from_mbox(filepath)
+    def from_mbox(cls, name: str, filepath: str) -> "ListservMailList":
+        """Docstring in `AbstractMailList`."""
+        msgs = bio.mlist_from_mbox(filepath)
         return cls(name, filepath, msgs)
 
     @classmethod
@@ -656,7 +465,7 @@ def from_listserv_directories(
         directorypaths: List[str],
         filedsc: str = "*.LOG?????",
         select: Optional[dict] = None,
-    ) -> "ListservList":
+    ) -> "ListservMailList":
         """
         This method is required if the files that contain the list messages
         were directly exported from LISTSERV 16.5 (e.g. by a member of 3GPP).
@@ -689,13 +498,13 @@ def from_listserv_files(
         name: str,
         filepaths: List[str],
         select: Optional[dict] = None,
-    ) -> "ListservList":
+    ) -> "ListservMailList":
         """
         This method is required if the files that contain the list messages
         were directly exported from LISTSERV 16.5 (e.g. by a member of 3GPP).
         Each mailing list has its own directory and is split over multiple
         files with an extension starting with LOG and ending with five digits.
-        Compared to `ListservList.from_listserv_directories()`, this method
+        Compared to `ListservMailList.from_listserv_directories()`, this method
         reads messages from single files, instead of all the files contained in
         a directory.
 
@@ -733,47 +542,6 @@ def from_listserv_files(
                 )
         return cls(name, filepaths, msgs)
 
-    @classmethod
-    def get_messages_from_url(
-        cls,
-        name: str,
-        url: str,
-        select: Optional[dict] = None,
-        session: Optional[dict] = None,
-    ) -> List[mboxMessage]:
-        """
-        Generator that returns all messages within a certain period
-        (e.g. January 2021, Week 5).
-
-        Parameters
-        ----------
-        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'
-        url : URL to the LISTSERV list.
-        select : Selection criteria that can filter messages by:
-            - content, i.e. header and/or body
-            - period, i.e. written in a certain year, month, week-of-month
-        session : requests.Session() object for the LISTSERV Email archive website.
-        """
-        # get all message URLs
-        msg_urls = cls.get_message_urls(name, url, select)
-        msg_parser = ListservMessageParser(website=True, session=session)
-        # get all message contents
-        msgs = []
-        for msg_url in tqdm(msg_urls, ascii=True, desc=name):
-            msg = msg_parser.from_url(
-                list_name=name,
-                url=msg_url,
-                fields=select["fields"],
-            )
-            if msg.get_payload() == "RequestException":
-                time.sleep(30)
-            else:
-                msgs.append(msg)
-                logger.info(f"Recorded the message {msg_url}.")
-            # wait between loading messages, for politeness
-            time.sleep(1)
-        return msgs
-
     @classmethod
     def get_message_urls(
         cls,
@@ -781,26 +549,21 @@ def get_message_urls(
         url: str,
         select: Optional[dict] = None,
     ) -> List[str]:
-        """
-        Parameters
-        ----------
-        name : Name of the list of messages, e.g. '3GPP_TSG_SA_WG2_UPCON'
-        url : URL to the LISTSERV list.
-        select : Selection criteria that can filter messages by:
-            - content, i.e. header and/or body
-            - period, i.e. written in a certain year, month, week-of-month
+        """Docstring in `AbstractMailList`."""
+
+        def get_message_urls_from_period_url(name: str, url: str) -> List[str]:
+            url_root = ("/").join(url.split("/")[:-2])
+            soup = get_website_content(url)
+            a_tags = soup.select(f'a[href*="A2="][href*="{name}"]')
+            if a_tags:
+                a_tags = [urljoin(url_root, url.get("href")) for url in a_tags]
+            return a_tags
 
-        Returns
-        -------
-        List of all selected URLs of the messages in the mailing list.
-        """
-        if select is None:
-            select = {"fields": "total"}
         msg_urls = []
         # run through periods
-        for period_url in ListservList.get_period_urls(url, select):
+        for period_url in ListservMailList.get_period_urls(url, select):
             # run through messages within period
-            for msg_url in ListservList.get_messages_urls(name, period_url):
+            for msg_url in get_message_urls_from_period_url(name, period_url):
                 msg_urls.append(msg_url)
         return msg_urls
 
@@ -838,7 +601,7 @@ def get_period_urls(
 
                 periodquants = [cond(period) for period in periods]
 
-                indices = ListservList.get_index_of_elements_in_selection(
+                indices = ListservMailList.get_index_of_elements_in_selection(
                     periodquants,
                     urls_of_periods,
                     value,
@@ -855,6 +618,11 @@ def get_all_periods_and_their_urls(
         """
         LISTSERV groups messages into weekly time bundles. This method
         obtains all the URLs that lead to the messages of each time bundle.
+
+        Returns
+        -------
+        Returns a tuple of two lists that look like:
+        (['April 2017,  2', 'January 2001', ...], ['ulr1', 'url2', ...])
         """
         # wait between loading messages, for politeness
         time.sleep(0.5)
@@ -868,70 +636,11 @@ def get_all_periods_and_their_urls(
         ]
         return periods, urls_of_periods
 
-    @staticmethod
-    def get_index_of_elements_in_selection(
-        times: List[Union[int, str]],
-        urls: List[str],
-        filtr: Union[tuple, list, int, str],
-    ) -> List[int]:
-        """
-        Filter out messages that where in a specific period. Period here is a set
-        containing units of year, month, and week-of-month which can have the following
-        example elements:
-            - years: (1992, 2010), [2000, 2008], 2021
-            - months: ["January", "July"], "November"
-            - weeks: (1, 4), [1, 5], 2
-
-        Parameters
-        ----------
-        times : A list containing information of the period for each
-            group of mboxMessage.
-        urls : Corresponding URLs of each group of mboxMessage of which the
-            period info is contained in `times`.
-        filtr : Containing info on what should be filtered.
-
-        Returns
-        -------
-        Indices of to the elements in `times`/`ursl`.
-        """
-        if isinstance(filtr, tuple):
-            # filter year or week in range
-            cond = lambda x: (np.min(filtr) <= x <= np.max(filtr))
-        if isinstance(filtr, list):
-            # filter in year, week, or month in list
-            cond = lambda x: x in filtr
-        if isinstance(filtr, int):
-            # filter specific year or week
-            cond = lambda x: x == filtr
-        if isinstance(filtr, str):
-            # filter specific month
-            cond = lambda x: x == filtr
-        return [idx for idx, time in enumerate(times) if cond(time)]
-
     @staticmethod
     def get_name_from_url(url: str) -> str:
         """Get name of mailing list."""
         return url.split("A0=")[-1]
 
-    @classmethod
-    def get_messages_urls(cls, name: str, url: str) -> List[str]:
-        """
-        Parameters
-        ----------
-        name : Name of the `ListservList`
-        url : URL to group of messages that are within the same period.
-
-        Returns
-        -------
-        List to URLs from which `mboxMessage` can be initialized.
-        """
-        url_root = ("/").join(url.split("/")[:-2])
-        soup = get_website_content(url)
-        a_tags = soup.select(f'a[href*="A2="][href*="{name}"]')
-        if a_tags:
-            a_tags = [urljoin(url_root, url.get("href")) for url in a_tags]
-        return a_tags
-
     @classmethod
     def get_line_numbers_of_header_starts(
         cls, content: List[str]
@@ -952,73 +661,36 @@ def get_line_numbers_of_header_starts(
             line_nr for line_nr, line in enumerate(content) if "=" * 73 in line
         ]
 
-    def to_dict(self, include_body: bool = True) -> Dict[str, List[str]]:
-        """
-        Parameters
-        ----------
-        include_body : A boolean that indicates whether the message body should
-            be included or not.
-
-        Returns
-        -------
-        A Dictionary with the first key layer being the header field names and
-        the "body" key. Each value field is a list containing the respective
-        header field contents arranged by the order as they were scraped from
-        the web. This format makes the conversion to a pandas.DataFrame easier.
-        """
-        return ListservListIO.to_dict(self.messages, include_body)
-
-    def to_pandas_dataframe(self, include_body: bool = True) -> pd.DataFrame:
-        """
-        Parameters
-        ----------
-        include_body : A boolean that indicates whether the message body should
-            be included or not.
 
-        Returns
-        -------
-        Converts the mailing list into a pandas.DataFrame object in which each
-        row represents an Email.
-        """
-        return ListservListIO.to_pandas_dataframe(self.messages, include_body)
-
-    def to_mbox(self, dir_out: str, filename: Optional[str] = None):
-        """Safe mailing list to .mbox files."""
-        if filename is None:
-            ListservListIO.to_mbox(self.messages, dir_out, self.name)
-        else:
-            ListservListIO.to_mbox(self.messages, dir_out, filename)
-
-
-class ListservArchive(object):
+class ListservMailListDomain(AbstractMailListDomain):
     """
-    This class handles the scraping of a public mailing list archive that uses
-    the LISTSERV 16.5 and 17 format.
-    An archive is a list of ListservList elements.
+    This class handles the scraping of a all public Emails contained in a mail
+    list domain that has the LISTSERV 16.5 and 17 format, such as 3GPP.
+    To be more precise, each contributor to a mail list domain sends their message
+    to an Email address that has the following structure:
+    <mailing_list_name>@w3.org.
+    Thus, this class contains all Emails send to <mail_list_domain_name>
+    (the Email domain name). These Emails are contained in a list of
+    `ListservMailList` types, such that it is known to which <mailing_list_name>
+    (the Email localpart) was send.
 
     Parameters
     ----------
-    name : The of whom the archive is (e.g. 3GPP, IEEE, ...)
-    url : The URL where the archive lives
-    lists : A list containing the mailing lists as `ListservList` types
+    name : The mailing list domain name (e.g. 3GPP, IEEE, ...)
+    url : The URL where the mailing list domain lives
+    lists : A list containing the mailing lists as `ListservMailList` types
 
     Methods
     -------
-    from_url()
-    from_mbox()
-    from_mailing_lists()
+    All methods in the `AbstractMailListDomain` class in addition to:
     from_listserv_directory()
-    get_lists()
     get_sections()
-    to_dict()
-    to_pandas_dataframe()
-    to_mbox()
 
     Example
     -------
-    To scrape a Listserv mailing list archive from an URL and store it in
+    To scrape a Listserv mailing list domain from an URL and store it in
     run-time memory, we do the following
-    >>> arch = ListservArchive.from_url(
+    >>> mlistdom = ListservMailListDomain.from_url(
     >>>     name="IEEE",
     >>>     url_root="https://listserv.ieee.org/cgi-bin/wa?",
     >>>     url_home="https://listserv.ieee.org/cgi-bin/wa?HOME",
@@ -1034,28 +706,9 @@ class ListservArchive(object):
     >>> )
 
     To save it as *.mbox file we do the following
-    >>> arch.to_mbox(path_to_directory)
+    >>> mlistdom.to_mbox(path_to_directory)
     """
 
-    def __init__(
-        self, name: str, url: str, lists: List[Union[ListservList, str]]
-    ):
-        self.name = name
-        self.url = url
-        self.lists = lists
-
-    def __len__(self):
-        """Get number of mailing lists within the mailing archive."""
-        return len(self.lists)
-
-    def __iter__(self):
-        """Iterate over each mailing list within the mailing archive."""
-        return iter(self.lists)
-
-    def __getitem__(self, index):
-        """Get specific mailing list at position `index` from the mailing archive."""
-        return self.lists[index]
-
     @classmethod
     def from_url(
         cls,
@@ -1069,34 +722,9 @@ def from_url(
         session: Optional[str] = None,
         instant_save: bool = True,
         only_mlist_urls: bool = True,
-    ) -> "ListservArchive":
-        """
-        Create ListservArchive from a given URL.
-
-        Parameters
-        ----------
-        name : Email archive name, such that multiple instances of `ListservArchive`
-            can easily be distinguished.
-        url_root : The invariant root URL that does not change no matter what
-            part of the LISTSERV archive we access.
-        url_home : The 'home' space of the LISTSERV archive. This is required as
-            it contains the different sections which we obtain using `get_sections()`.
-        select: Selection criteria that can filter messages by:
-            - content, i.e. header and/or body
-            - period, i.e. written in a certain year, month, week-of-month
-        url_login : URL to the 'Log In' page.
-        url_pref : URL to the 'Preferences'/settings page.
-        login : Login credentials (username and password) that were used to set
-            up AuthSession. You can create your own for the 3GPP archive.
-        session : requests.Session() object for the LISTSERV Email archive website.
-        instant_save : Boolean giving the choice to save a `ListservList` as
-            soon as it is completely scraped or collect entire archive. The
-            prior is recommended if a large number of mailing lists are
-            scraped which can require a lot of memory and time.
-        only_list_urls : Boolean giving the choice to collect only `ListservList`
-            URLs or also their contents.
-        """
-        if session is None:
+    ) -> "ListservMailListDomain":
+        """Docstring in `AbstractMailList`."""
+        if (session is None) and (url_login is not None):
             session = get_auth_session(url_login, **login)
             session = set_website_preference_for_header(url_pref, session)
         lists = cls.get_lists_from_url(
@@ -1121,7 +749,7 @@ def from_mailing_lists(
         cls,
         name: str,
         url_root: str,
-        url_mailing_lists: Union[List[str], List[ListservList]],
+        url_mailing_lists: Union[List[str], List[ListservMailList]],
         select: Optional[dict] = None,
         url_login: str = "https://list.etsi.org/scripts/wa.exe?LOGON",
         url_pref: str = "https://list.etsi.org/scripts/wa.exe?PREF",
@@ -1129,39 +757,16 @@ def from_mailing_lists(
         session: Optional[str] = None,
         only_mlist_urls: bool = True,
         instant_save: Optional[bool] = True,
-    ) -> "ListservArchive":
-        """
-        Create ListservArchive from a given list of 'ListservList'.
-
-        Parameters
-        ----------
-        name : Email archive name, such that multiple instances of `ListservArchive`
-            can easily be distinguished.
-        url_root : The invariant root URL that does not change no matter what
-            part of the LISTSERV archive we access.
-        url_mailing_lists : This argument can either be a list of `ListservList`
-            objects or a list of string containing the URLs to the LISTSERV
-            Email lists of interest.
-        url_login : URL to the 'Log In' page.
-        url_pref : URL to the 'Preferences'/settings page.
-        login : Login credentials (username and password) that were used to set
-            up AuthSession. You can create your own for the 3GPP archive.
-        session : requests.Session() object for the LISTSERV Email archive website.
-        only_list_urls : Boolean giving the choice to collect only `ListservList`
-            URLs or also their contents.
-        instant_save : Boolean giving the choice to save a `ListservList` as
-            soon as it is completely scraped or collect entire archive. The
-            prior is recommended if a large number of mailing lists are
-            scraped which can require a lot of memory and time.
-        """
+    ) -> "ListservMailListDomain":
+        """Docstring in `AbstractMailList`."""
         if isinstance(url_mailing_lists[0], str) and only_mlist_urls is False:
-            if session is None:
+            if (session is None) and (url_login is not None):
                 session = get_auth_session(url_login, **login)
                 session = set_website_preference_for_header(url_pref, session)
             lists = []
             for url in url_mailing_lists:
-                mlist_name = url.split("A0=")[-1]
-                mlist = ListservList.from_url(
+                mlist_name = ListservMailList.get_name_from_url(url)
+                mlist = ListservMailList.from_url(
                     name=mlist_name,
                     url=url,
                     select=select,
@@ -1187,18 +792,18 @@ def from_listserv_directory(
         folderdsc: str = "*",
         filedsc: str = "*.LOG?????",
         select: Optional[dict] = None,
-    ) -> "ListservArchive":
+    ) -> "ListservMailListDomain":
         """
-        This method is required if the files that contain the archive messages
+        This method is required if the files that contain the mail list domain messages
         were directly exported from LISTSERV 16.5 (e.g. by a member of 3GPP).
         Each mailing list has its own subdirectory and is split over multiple
         files with an extension starting with LOG and ending with five digits.
 
         Parameters
         ----------
-        name : Email archive name, such that multiple instances of `ListservArchive`
-            can easily be distinguished.
-        directorypath : Where the `ListservArchive` can be initialised.
+        name : mail list domain name, such that multiple instances of
+            `ListservMailListDomain` can easily be distinguished.
+        directorypath : Where the `ListservMailListDomain` can be initialised.
         folderdsc : A description of the relevant folders
         filedsc : A description of the relevant files, e.g. *.LOG?????
         select : Selection criteria that can filter messages by:
@@ -1212,7 +817,7 @@ def from_listserv_directory(
         # run through directories and collect all filepaths
         for dirpath in _dirpaths_to_lists:
             _filepaths = get_paths_to_files_in_directory(dirpath, filedsc)
-            mlist = ListservList.from_listserv_files(
+            mlist = ListservMailList.from_listserv_files(
                 dirpath.split("/")[-2],
                 _filepaths,
                 select,
@@ -1226,21 +831,13 @@ def from_mbox(
         name: str,
         directorypath: str,
         filedsc: str = "*.mbox",
-    ) -> "ListservArchive":
-        """
-        Parameters
-        ----------
-        name : Email archive name, such that multiple instances of `ListservArchive`
-            can easily be distinguished.
-        directorypath : Path to the folder in which `ListservArchive` is stored.
-        filedsc : Optional filter that only reads files matching the description.
-            By default all files with an mbox extension are read.
-        """
+    ) -> "ListservMailList":
+        """Docstring in `AbstractMailList`."""
         filepaths = get_paths_to_files_in_directory(directorypath, filedsc)
         lists = []
         for filepath in filepaths:
             name = filepath.split("/")[-1].split(".")[0]
-            lists.append(ListservList.from_mbox(name, filepath))
+            lists.append(ListservMailList.from_mbox(name, filepath))
         return cls(name, directorypath, lists)
 
     @staticmethod
@@ -1251,35 +848,12 @@ def get_lists_from_url(
         session: Optional[str] = None,
         instant_save: bool = True,
         only_mlist_urls: bool = True,
-    ) -> List[Union[ListservList, str]]:
-        """
-        Created dictionary of all lists in the archive.
-
-        Parameters
-        ----------
-        url_root : The invariant root URL that does not change no matter what
-            part of the LISTSERV archive we access.
-        url_home : The 'home' space of the LISTSERV archive. This is required as
-            it contains the different sections which we obtain using `get_sections()`.
-        select : Selection criteria that can filter messages by:
-            - content, i.e. header and/or body
-            - period, i.e. written in a certain year, month, week-of-month
-        session : requests.Session() object for the LISTSERV Email archive website.
-        instant_save : Boolean giving the choice to save a `ListservList` as
-            soon as it is completely scraped or collect entire archive. The
-            prior is recommended if a large number of mailing lists are
-            scraped which can require a lot of memory and time.
-        only_list_urls : Boolean giving the choice to collect only `ListservList`
-            URLs or also their contents.
-
-        Returns
-        -------
-        archive_dict : the keys are the names of the lists and the value their url
-        """
+    ) -> List[Union[ListservMailList, str]]:
+        """Docstring in `AbstractMailList`."""
         archive = []
         # run through archive sections
         for url in list(
-            ListservArchive.get_sections(url_root, url_home).keys()
+            ListservMailListDomain.get_sections(url_root, url_home).keys()
         ):
             soup = get_website_content(url)
             a_tags_in_section = soup.select(
@@ -1295,18 +869,20 @@ def get_lists_from_url(
             if only_mlist_urls:
                 # collect mailing-list urls
                 for mlist_url in mlist_urls:
-                    name = ListservList.get_name_from_url(mlist_url)
+                    name = ListservMailList.get_name_from_url(mlist_url)
                     # check if mailing list contains messages in period
-                    _period_urls = ListservList.get_all_periods_and_their_urls(
-                        mlist_url
-                    )[1]
+                    _period_urls = (
+                        ListservMailList.get_all_periods_and_their_urls(
+                            mlist_url
+                        )[1]
+                    )
                     # check if mailing list is public
                     if len(_period_urls) > 0:
                         loops = 0
                         for _period_url in _period_urls:
                             loops += 1
                             nr_msgs = len(
-                                ListservList.get_messages_urls(
+                                ListservMailList.get_messages_urls(
                                     name=name, url=_period_url
                                 )
                             )
@@ -1316,8 +892,8 @@ def get_lists_from_url(
             else:
                 # collect mailing-list contents
                 for mlist_url in mlist_urls:
-                    name = ListservList.get_name_from_url(mlist_url)
-                    mlist = ListservList.from_url(
+                    name = ListservMailList.get_name_from_url(mlist_url)
+                    mlist = ListservMailList.from_url(
                         name=name,
                         url=mlist_url,
                         select=select,
@@ -1336,7 +912,7 @@ def get_lists_from_url(
 
     def get_sections(url_root: str, url_home: str) -> int:
         """
-        Get different sections of archive.
+        Get different sections of mail list domain.
         On the Listserv 16.5 website they look like:
         [3GPP] [3GPP–AT1] [AT2–CONS] [CONS–EHEA] [EHEA–ERM_] ...
         On the Listserv 17 website they look like:
@@ -1363,190 +939,3 @@ def get_sections(url_root: str, url_home: str) -> int:
         else:
             archive_sections_dict[url_home] = "Home"
         return archive_sections_dict
-
-    def to_conversationkg_dict(self) -> Dict[str, List[str]]:
-        """
-        Place all message in all lists into a dictionary of the form:
-            dic = {
-                "message_ID1": {
-                    "body": ...,
-                    "subject": ...,
-                    ... ,
-                }
-                "message_ID2": {
-                    "body": ...,
-                    "subject": ...,
-                    ... ,
-                }
-            }
-        """
-        # initialize dictionary
-        dic = {}
-        msg_nr = 0
-        # run through lists
-        for mlist in self.lists:
-            # run through messages
-            for msg in mlist.messages:
-                dic[f"ID{msg_nr}"] = msg.to_dict()
-                msg_nr += 1
-        return dic
-
-    def to_dict(self, include_body: bool = True) -> Dict[str, List[str]]:
-        """
-        Concatenates mailing list dictionaries created using
-        `ListservList.to_dict()`.
-        """
-        return ListservArchiveIO.to_dict(self.lists, include_body)
-
-    def to_pandas_dataframe(self, include_body: bool = True) -> pd.DataFrame:
-        """
-        Concatenates mailing list pandas.DataFrames created using
-        `ListservList.to_pandas_dataframe()`.
-        """
-        return ListservArchiveIO.to_pandas_dataframe(self.lists, include_body)
-
-    def to_mbox(self, dir_out: str):
-        """
-        Save Archive content to .mbox files
-        """
-        ListservArchiveIO.to_mbox(self.lists, dir_out)
-
-
-def set_website_preference_for_header(
-    url_pref: str,
-    session: requests.Session,
-) -> requests.Session:
-    """
-    Set the 'Email Headers' of the 'Archive Preferences' for the auth session
-    to 'Show All Headers'. Otherwise only a restricted list of header fields is
-    shown.
-    """
-    url_archpref = url_pref + "&TAB=2"
-    payload = {
-        "Email Headers": "b",
-    }
-    session.post(url_archpref, data=payload)
-    return session
-
-
-def get_auth_session(
-    url_login: str, username: str, password: str
-) -> requests.Session:
-    """
-    Create AuthSession.
-
-    There are three ways to create an AuthSession:
-        - parse username & password directly into method
-        - create a /bigbang/config/authentication.yaml file that contains keys
-        - type then into terminal when the method 'get_login_from_terminal'
-            is raised
-    """
-    if os.path.isfile(filepath_auth):
-        # read from /config/authentication.yaml
-        with open(filepath_auth, "r") as stream:
-            auth_key = yaml.safe_load(stream)
-        username = auth_key["username"]
-        password = auth_key["password"]
-    else:
-        # ask user for login keys
-        username, password = get_login_from_terminal(username, password)
-
-    if username is None or password is None:
-        # continue without authentication
-        return None
-    else:
-        # Start the AuthSession
-        session = requests.Session()
-        # Create the payload
-        payload = {
-            "LOGIN1": "",
-            "Y": username,
-            "p": password,
-            "X": "",
-        }
-        # Post the payload to the site to log in
-        session.post(url_login, data=payload)
-        return session
-
-
-def get_login_from_terminal(
-    username: Union[str, None],
-    password: Union[str, None],
-    file_auth: str = directory_project + "/config/authentication.yaml",
-) -> Tuple[Union[str, None]]:
-    """
-    Get login key from user during run time if 'username' and/or 'password' is 'None'.
-    Return 'None' if no reply within 15 sec.
-    """
-    if username is None or password is None:
-        record = True
-    else:
-        record = False
-    if username is None:
-        username = ask_for_input("Enter your Email: ")
-    if password is None:
-        password = ask_for_input("Enter your Password: ")
-    if record and isinstance(username, str) and isinstance(password, str):
-        loginkey_to_file(username, password, file_auth)
-    return username, password
-
-
-def ask_for_input(request: str) -> Union[str, None]:
-    timeout = 15
-    end_time = time.time() + timeout
-    while time.time() < end_time:
-        reply = input(request)
-        try:
-            assert isinstance(reply, str)
-            break
-        except Exception:
-            reply = None
-            continue
-    return reply
-
-
-def loginkey_to_file(
-    username: str,
-    password: str,
-    file_auth: str,
-) -> None:
-    """Safe login key to yaml"""
-    file = open(file_auth, "w")
-    file.write(f"username: '{username}'\n")
-    file.write(f"password: '{password}'")
-    file.close()
-
-
-def get_website_content(
-    url: str,
-    session: Optional[requests.Session] = None,
-) -> Union[str, BeautifulSoup]:
-    """
-    Get HTML code from website
-
-    Note
-    ----
-    Servers don't like it when one is sending too many requests from same
-    ip address in short period of time. Therefore we need to:
-        a) catch 'requests.exceptions.RequestException' errors
-            (includes all possible errors to be on the safe side),
-        b) safe intermediate results,
-        c) continue where we left off at a later stage.
-    """
-    # TODO: include option to change BeautifulSoup args
-    try:
-        if session is None:
-            sauce = requests.get(url)
-            assert sauce.status_code == 200
-            soup = BeautifulSoup(sauce.content, "html.parser")
-        else:
-            sauce = session.get(url)
-            soup = BeautifulSoup(sauce.text, "html.parser")
-        return soup
-    except requests.exceptions.RequestException as e:
-        if "A2=" in url:
-            # if URL of mboxMessage
-            logger.info(f"{e} for {url}.")
-            return "RequestException"
-        else:
-            SystemExit()
diff --git a/bigbang/ingress/mailman.py b/bigbang/ingress/mailman.py
index e9d1de4e..cd8015ab 100644
--- a/bigbang/ingress/mailman.py
+++ b/bigbang/ingress/mailman.py
@@ -24,7 +24,8 @@
 
 from config.config import CONFIG
 
-from .. import listserv, parse, w3crawl
+from . import listserv, w3c
+from .. import parse
 
 ml_exp = re.compile(r"/([\w-]+)/?$")
 txt_exp = re.compile(r'href="(\d\d\d\d-\w*\.txt)"')
@@ -247,9 +248,15 @@ def collect_archive_from_url(
         url_root = "https://" + urlparse(url).hostname
 
     if w3c_archives_exp.search(url):
-        return w3crawl.collect_from_url(url, archive_dir, notes=notes)
+        return w3c.W3CMailListDomain.from_mailing_lists(
+            name="W3C",
+            url_root=url_root,
+            url_mailing_lists=urls,
+            only_mlist_urls=False,
+            instant_save=True,
+        )
     elif tgpp_archives_exp.search(url):
-        return listserv.ListservArchive.from_mailing_lists(
+        return listserv.ListservMailListDomain.from_mailing_lists(
             name="3GPP",
             url_root=url_root,
             url_mailing_lists=urls,
@@ -257,7 +264,7 @@ def collect_archive_from_url(
             instant_save=True,
         )
     elif ieee_archives_exp.search(url):
-        return listserv.ListservArchive.from_mailing_lists(
+        return listserv.ListservMailListDomain.from_mailing_lists(
             name="IEEE",
             url_root=url_root,
             url_mailing_lists=urls,
diff --git a/bigbang/ingress/utils.py b/bigbang/ingress/utils.py
new file mode 100644
index 00000000..cc15e845
--- /dev/null
+++ b/bigbang/ingress/utils.py
@@ -0,0 +1,169 @@
+"""
+Miscellaneous utility functions used in other modules.
+"""
+import os
+import time
+import logging
+import glob
+import yaml
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+from bs4 import BeautifulSoup
+import requests
+
+import networkx as nx
+import pandas as pd
+
+from config.config import CONFIG
+
+filepath_auth = CONFIG.config_path + "authentication.yaml"
+directory_project = str(Path(os.path.abspath(__file__)).parent.parent)
+logging.basicConfig(
+    filename=directory_project + "/scraping.log",
+    filemode="w",
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def get_website_content(
+    url: str,
+    session: Optional[requests.Session] = None,
+) -> Union[str, BeautifulSoup]:
+    """
+    Get HTML code from website
+
+    Note
+    ----
+    Servers don't like it when one is sending too many requests from same
+    ip address in short period of time. Therefore we need to:
+        a) catch 'requests.exceptions.RequestException' errors
+            (includes all possible errors to be on the safe side),
+        b) safe intermediate results,
+        c) continue where we left off at a later stage.
+    """
+    # TODO: include option to change BeautifulSoup args
+    try:
+        if session is None:
+            sauce = requests.get(url)
+            if sauce.status_code == 200:
+                soup = BeautifulSoup(sauce.content, "html.parser")
+            elif sauce.status_code == 404:
+                return "RequestException"
+        else:
+            sauce = session.get(url)
+            soup = BeautifulSoup(sauce.text, "html.parser")
+        return soup
+    except requests.exceptions.RequestException as e:
+        if "A2=" in url:
+            # if URL of mboxMessage
+            logger.info(f"{e} for {url}.")
+            return "RequestException"
+        else:
+            SystemExit()
+
+
+def set_website_preference_for_header(
+    url_pref: str,
+    session: requests.Session,
+) -> requests.Session:
+    """
+    Set the 'Email Headers' of the 'Archive Preferences' for the auth session
+    to 'Show All Headers'. Otherwise only a restricted list of header fields is
+    shown.
+    """
+    url_archpref = url_pref + "&TAB=2"
+    payload = {
+        "Email Headers": "b",
+    }
+    session.post(url_archpref, data=payload)
+    return session
+
+
+def get_auth_session(
+    url_login: str, username: str, password: str
+) -> requests.Session:
+    """
+    Create AuthSession.
+
+    There are three ways to create an AuthSession:
+        - parse username & password directly into method
+        - create a /bigbang/config/authentication.yaml file that contains keys
+        - type then into terminal when the method 'get_login_from_terminal'
+            is raised
+    """
+    if os.path.isfile(filepath_auth):
+        # read from /config/authentication.yaml
+        with open(filepath_auth, "r") as stream:
+            auth_key = yaml.safe_load(stream)
+        username = auth_key["username"]
+        password = auth_key["password"]
+    else:
+        # ask user for login keys
+        username, password = get_login_from_terminal(username, password)
+
+    if username is None or password is None:
+        # continue without authentication
+        return None
+    else:
+        # Start the AuthSession
+        session = requests.Session()
+        # Create the payload
+        payload = {
+            "LOGIN1": "",
+            "Y": username,
+            "p": password,
+            "X": "",
+        }
+        # Post the payload to the site to log in
+        session.post(url_login, data=payload)
+        return session
+
+
+def get_login_from_terminal(
+    username: Union[str, None],
+    password: Union[str, None],
+    file_auth: str = directory_project + "/config/authentication.yaml",
+) -> Tuple[Union[str, None]]:
+    """
+    Get login key from user during run time if 'username' and/or 'password' is 'None'.
+    Return 'None' if no reply within 15 sec.
+    """
+    if username is None or password is None:
+        record = True
+    else:
+        record = False
+    if username is None:
+        username = ask_for_input("Enter your Email: ")
+    if password is None:
+        password = ask_for_input("Enter your Password: ")
+    if record and isinstance(username, str) and isinstance(password, str):
+        loginkey_to_file(username, password, file_auth)
+    return username, password
+
+
+def ask_for_input(request: str) -> Union[str, None]:
+    timeout = 15
+    end_time = time.time() + timeout
+    while time.time() < end_time:
+        reply = input(request)
+        try:
+            assert isinstance(reply, str)
+            break
+        except Exception:
+            reply = None
+            continue
+    return reply
+
+
+def loginkey_to_file(
+    username: str,
+    password: str,
+    file_auth: str,
+) -> None:
+    """Safe login key to yaml"""
+    file = open(file_auth, "w")
+    file.write(f"username: '{username}'\n")
+    file.write(f"password: '{password}'")
+    file.close()
diff --git a/bigbang/ingress/w3c.py b/bigbang/ingress/w3c.py
new file mode 100644
index 00000000..0a3baf39
--- /dev/null
+++ b/bigbang/ingress/w3c.py
@@ -0,0 +1,574 @@
+import email
+import logging
+import os
+import re
+import subprocess
+import time
+import warnings
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+from urllib.parse import urljoin, urlparse
+
+import numpy as np
+import pandas as pd
+import requests
+import yaml
+from bs4 import BeautifulSoup
+
+from config.config import CONFIG
+
+import bigbang.bigbang_io as bio
+from bigbang.data_types import MailList
+from bigbang.ingress import (
+    AbstractMessageParser,
+    AbstractMailList,
+    AbstractMailListDomain,
+)
+from bigbang.ingress.utils import (
+    get_website_content,
+    set_website_preference_for_header,
+    get_auth_session,
+)
+from bigbang.utils import (
+    get_paths_to_files_in_directory,
+    get_paths_to_dirs_in_directory,
+)
+
+filepath_auth = CONFIG.config_path + "authentication.yaml"
+directory_project = str(Path(os.path.abspath(__file__)).parent.parent)
+logging.basicConfig(
+    filename=directory_project + "/w3c.scraping.log",
+    filemode="w",
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+class W3CMessageParserWarning(BaseException):
+    """Base class for W3CMessageParser class specific exceptions"""
+
+    pass
+
+
+class W3CMailListWarning(BaseException):
+    """Base class for W3CMailList class specific exceptions"""
+
+    pass
+
+
+class W3CMailListDomainWarning(BaseException):
+    """Base class for W3CMailListDomain class specific exceptions"""
+
+    pass
+
+
+class W3CMessageParser(AbstractMessageParser, email.parser.Parser):
+    """
+    This class handles the creation of an `mailbox.mboxMessage` object
+    (using the from_*() methods) and its storage in various other file formats
+    (using the to_*() methods) that can be saved on the local memory.
+
+    Parameters
+    ----------
+    website : Set 'True' if messages are going to be scraped from websites,
+        otherwise 'False' if read from local memory. This distinction needs to
+        be made if missing messages should be added.
+    url_pref : URL to the 'Preferences'/settings page.
+
+    Methods
+    -------
+    from_url()
+    _get_header_from_html()
+    _get_body_from_html()
+
+    Example
+    -------
+    To create a Email message parser object, use the following syntax:
+    >>> msg_parser = W3CMessageParser(website=True)
+
+    To obtain the Email message content and return it as `mboxMessage` object,
+    you need to do the following:
+    >>> msg = msg_parser.from_url(
+    >>>     list_name="public-2018-permissions-ws",
+    >>>     url="https://lists.w3.org/Archives/Public/public-2018-permissions-ws/2019May/0000.html",
+    >>>     fields="total",
+    >>> )
+    """
+
+    empty_header = {}
+
+    def _get_header_from_html(self, soup: BeautifulSoup) -> Dict[str, str]:
+        """
+        Lexer for the message header.
+
+        Parameters
+        ----------
+        soup : HTML code from which the Email header can be obtained.
+        """
+        header = {
+            "message-ID": "#message-id",
+            "Date": "#date",
+            "To": "#to",
+            "Cc": "#cc",
+        }
+        for key, value in header.items():
+            try:
+                header[key] = parse_dfn_header(
+                    text_for_selector(soup, value)
+                ).strip()
+            except Exception:
+                header[key] = ""
+                continue
+        header["Subject"] = text_for_selector(soup, "h1")
+
+        from_text = parse_dfn_header(text_for_selector(soup, "#from"))
+        from_name = from_text.split("<")[0].strip()
+        from_address = text_for_selector(soup, "#from a")
+        header["From"] = email.utils.formataddr(
+            (from_name, email.header.Header(from_address).encode())
+        )
+
+        in_reply_to_pattern = re.compile('<!-- inreplyto="(.+?)"')
+        match = in_reply_to_pattern.search(str(soup))
+        if match:
+            header["In-Reply-To"] = "<" + match.groups()[0] + ">"
+
+        return header
+
+    def _get_body_from_html(
+        self, list_name: str, url: str, soup: BeautifulSoup
+    ) -> Union[str, None]:
+        """
+        Lexer for the message body/payload.
+        This methods assumes that the body is available in text/plain.
+
+        Parameters
+        ----------
+        url : URL to the Email message.
+        soup : HTML code from which the Email body can be obtained.
+        """
+        # TODO re-write using email.parser.Parser
+        try:
+            return text_for_selector(soup, "#body")
+        except Exception:
+            logger.info(f"The message body of {url} could not be loaded.")
+            return None
+
+
+class W3CMailList(AbstractMailList):
+    """
+    This class handles the scraping of a all public Emails contained in a single
+    mailing list in the hypermail format.
+    To be more precise, each contributor to a mailing list sends
+    their message to an Email address that has the following structure:
+    <mailing_list_name>@w3.org.
+    Thus, this class contains all Emails send to a specific <mailing_list_name>
+    (the Email localpart, such as "public-abcg" or "public-accesslearn-contrib").
+
+    Parameters
+    ----------
+    name : The name of the list (e.g. public-2018-permissions-ws, ...)
+    source : Contains the information of the location of the mailing list.
+        It can be either an URL where the list or a path to the file(s).
+    msgs : List of mboxMessage objects
+
+    Methods
+    -------
+    All methods in the `AbstractMailList` class in addition to:
+    get_all_periods_and_their_urls()
+
+
+    Example
+    -------
+    To scrape a W3C mailing list from an URL and store it in
+    run-time memory, we do the following
+    >>> mlist = W3CMailList.from_url(
+    >>>     name="public-bigdata",
+    >>>     url="https://lists.w3.org/Archives/Public/public-bigdata/",
+    >>>     select={
+    >>>         "years": 2015,
+    >>>         "months": "August",
+    >>>         "fields": "header",
+    >>>     },
+    >>> )
+
+    To save it as *.mbox file we do the following
+    >>> mlist.to_mbox(path_to_file)
+    """
+
+    @classmethod
+    def from_url(
+        cls,
+        name: str,
+        url: str,
+        select: Optional[dict] = {"fields": "total"},
+    ) -> "W3CMailList":
+        """Docstring in `AbstractMailList`."""
+        if "fields" not in list(select.keys()):
+            select["fields"] = "total"
+        msg_urls = cls.get_message_urls(name, url, select)
+        return cls.from_messages(
+            name,
+            url,
+            msg_urls,
+            select["fields"],
+        )
+
+    @classmethod
+    def from_messages(
+        cls,
+        name: str,
+        url: str,
+        messages: Union[List[str], MailList],
+        fields: str = "total",
+    ) -> "W3CMailList":
+        """Docstring in `AbstractMailList`."""
+        if not messages:
+            msgs = []
+        elif isinstance(messages[0], str):
+            msg_parser = W3CMessageParser(
+                website=True,
+            )
+            msgs = super().get_messages_from_urls(
+                name, messages, msg_parser, fields
+            )
+        else:
+            msgs = messages
+        return cls(name, url, msgs)
+
+    @classmethod
+    def from_mbox(cls, name: str, filepath: str) -> "W3CMailList":
+        """Docstring in `AbstractMailList`."""
+        msgs = bio.mlist_from_mbox(filepath)
+        return cls(name, filepath, msgs)
+
+    @classmethod
+    def get_message_urls(
+        cls,
+        name: str,
+        url: str,
+        select: Optional[dict] = None,
+    ) -> List[str]:
+        """Docstring in `AbstractMailList`."""
+
+        def get_message_urls_from_period_url(name: str, url: str) -> List[str]:
+            soup = get_website_content(url)
+            a_tags = soup.select("div.messages-list a")
+            if a_tags:
+                a_tags = [
+                    urljoin(url, a_tag.get("href"))
+                    for a_tag in a_tags
+                    if a_tag.get("href") is not None
+                ]
+            return a_tags
+
+        msg_urls = []
+        # run through periods
+        for period_url in W3CMailList.get_period_urls(url, select):
+            # run through messages within period
+            for msg_url in get_message_urls_from_period_url(name, period_url):
+                msg_urls.append(msg_url)
+        return msg_urls
+
+    @classmethod
+    def get_period_urls(
+        cls, url: str, select: Optional[dict] = None
+    ) -> List[str]:
+        """
+        All messages within a certain period (e.g. January 2021).
+
+        Parameters
+        ----------
+        url : URL to the W3C list.
+        select : Selection criteria that can filter messages by:
+            - content, i.e. header and/or body
+            - period, i.e. written in a certain year and month
+        """
+        # create dictionary with key indicating period and values the url
+        periods, urls_of_periods = cls.get_all_periods_and_their_urls(url)
+
+        if any(
+            period in list(select.keys()) for period in ["years", "months"]
+        ):
+            for key, value in select.items():
+                if key == "years":
+                    cond = lambda x: int(re.findall(r"\d{4}", x)[0])
+                elif key == "months":
+                    cond = lambda x: x.split(" ")[0]
+                else:
+                    continue
+
+                periodquants = [cond(period) for period in periods]
+
+                indices = W3CMailList.get_index_of_elements_in_selection(
+                    periodquants,
+                    urls_of_periods,
+                    value,
+                )
+
+                periods = [periods[idx] for idx in indices]
+                urls_of_periods = [urls_of_periods[idx] for idx in indices]
+        return urls_of_periods
+
+    @staticmethod
+    def get_all_periods_and_their_urls(
+        url: str,
+    ) -> Tuple[List[str], List[str]]:
+        """
+        W3C groups messages into monthly time bundles. This method
+        obtains all the URLs that lead to the messages of each time bundle.
+
+        Returns
+        -------
+        Returns a tuple of two lists that look like:
+        (['April 2017', 'January 2001', ...], ['ulr1', 'url2', ...])
+        """
+        # wait between loading messages, for politeness
+        time.sleep(0.5)
+        soup = get_website_content(url)
+        periods = []
+        urls_of_periods = []
+        rows = soup.select("tbody tr")
+        for row in rows:
+            link = row.select("td:nth-of-type(1) a")
+            if len(link) > 0:
+                link = link[0]
+            else:
+                continue
+            periods.append(link.text)
+            urls_of_periods.append(url + link.get("href"))
+        return periods, urls_of_periods
+
+    @classmethod
+    def get_messages_urls(cls, name: str, url: str) -> List[str]:
+        """
+        Parameters
+        ----------
+        name : Name of the W3C mailing list.
+        url : URL to group of messages that are within the same period.
+
+        Returns
+        -------
+        List of URLs from which `mboxMessage` can be initialized.
+        """
+        soup = get_website_content(url)
+        if soup == "RequestException":
+            return []
+        else:
+            a_tags = soup.select("div.messages-list a")
+            if a_tags:
+                a_tags = [
+                    urljoin(url, a_tag.get("href"))
+                    for a_tag in a_tags
+                    if a_tag.get("href") is not None
+                ]
+            return a_tags
+
+    @staticmethod
+    def get_name_from_url(url: str) -> str:
+        """Get name of mailing list."""
+        return url.split("/")[-2]
+
+
+class W3CMailListDomain(AbstractMailListDomain):
+    """
+    This class handles the scraping of a all public Emails contained in a mail
+    list domain that has the hypermail format, such as W3C.
+    To be more precise, each contributor to a mail list domain sends their message
+    to an Email address that has the following structure:
+    <mailing_list_name>@w3.org.
+    Thus, this class contains all Emails send to <mail_list_domain_name>
+    (the Email domain name). These Emails are contained in a list of
+    `W3CMailList` types, such that it is known to which <mailing_list_name>
+    (the Email localpart) was send.
+
+
+    Parameters
+    ----------
+    name : The name of the mailing list domain.
+    url : The URL where the mailing list domain lives
+    lists : A list containing the mailing lists as `W3CMailList` types
+
+    Methods
+    -------
+    All methods in the `AbstractMailListDomain` class.
+
+    Example
+    -------
+    To scrape a W3C mailing list mailing list domain from an URL and store it in
+    run-time memory, we do the following
+    >>> mlistdom = W3CMailListDomain.from_url(
+    >>>     name="W3C",
+    >>>     url_root="https://lists.w3.org/Archives/Public/",
+    >>>     select={
+    >>>         "years": 2015,
+    >>>         "months": "November",
+    >>>         "weeks": 4,
+    >>>         "fields": "header",
+    >>>     },
+    >>>     instant_save=False,
+    >>>     only_mlist_urls=False,
+    >>> )
+
+    To save it as *.mbox file we do the following
+    >>> mlistdom.to_mbox(path_to_directory)
+    """
+
+    @classmethod
+    def from_url(
+        cls,
+        name: str,
+        url_root: str,
+        url_home: Optional[str] = None,
+        select: Optional[dict] = None,
+        instant_save: bool = True,
+        only_mlist_urls: bool = True,
+    ) -> "W3CMailListDomain":
+        """Docstring in `AbstractMailListDomain`."""
+        lists = cls.get_lists_from_url(
+            select,
+            url_root,
+            url_home,
+            instant_save,
+            only_mlist_urls,
+        )
+        return cls.from_mailing_lists(
+            name,
+            url_root,
+            lists,
+            select,
+            only_mlist_urls,
+        )
+
+    @classmethod
+    def from_mailing_lists(
+        cls,
+        name: str,
+        url_root: str,
+        url_mailing_lists: Union[List[str], List[W3CMailList]],
+        select: Optional[dict] = None,
+        only_mlist_urls: bool = True,
+        instant_save: Optional[bool] = True,
+    ) -> "W3CMailListDomain":
+        """Docstring in `AbstractMailListDomain`."""
+        if isinstance(url_mailing_lists[0], str) and only_mlist_urls is False:
+            lists = []
+            for url in url_mailing_lists:
+                mlist_name = W3CMailList.get_name_from_url(url)
+                mlist = W3CMailList.from_url(
+                    name=mlist_name,
+                    url=url,
+                    select=select,
+                )
+                if len(mlist) != 0:
+                    if instant_save:
+                        dir_out = CONFIG.mail_path + name
+                        Path(dir_out).mkdir(parents=True, exist_ok=True)
+                        mlist.to_mbox(dir_out=dir_out)
+                    else:
+                        logger.info(f"Recorded the list {mlist.name}.")
+                        lists.append(mlist)
+        else:
+            lists = url_mailing_lists
+        return cls(name, url_root, lists)
+
+    @classmethod
+    def from_mbox(
+        cls,
+        name: str,
+        directorypath: str,
+        filedsc: str = "*.mbox",
+    ) -> "W3CMailListDomain":
+        """Docstring in `AbstractMailListDomain`."""
+        filepaths = get_paths_to_files_in_directory(directorypath, filedsc)
+        lists = []
+        for filepath in filepaths:
+            name = filepath.split("/")[-1].split(".")[0]
+            lists.append(W3CMailList.from_mbox(name, filepath))
+        return cls(name, directorypath, lists)
+
+    @staticmethod
+    def get_lists_from_url(
+        select: dict,
+        url_root: str,
+        url_home: Optional[str] = None,
+        instant_save: bool = True,
+        only_mlist_urls: bool = True,
+    ) -> List[Union[W3CMailList, str]]:
+        """Docstring in `AbstractMailListDomain`."""
+        archive = []
+        if url_home is None:
+            soup = get_website_content(url_root)
+        else:
+            soup = get_website_content(url_home)
+        mlist_urls = [
+            urljoin(url_root, h3_tag.select("a")[0].get("href"))
+            for h3_tag in soup.select("h3")
+            if h3_tag.select("a")
+        ]
+        mlist_urls = list(set(mlist_urls))  # remove duplicates
+
+        if only_mlist_urls:
+            # collect mailing-list urls
+            for mlist_url in mlist_urls:
+                name = W3CMailList.get_name_from_url(mlist_url)
+                # check if mailing list contains messages in period
+                _period_urls = W3CMailList.get_all_periods_and_their_urls(
+                    mlist_url
+                )[1]
+                # check if mailing list is public
+                if len(_period_urls) > 0:
+                    loops = 0
+                    for _period_url in _period_urls:
+                        loops += 1
+                        nr_msgs = len(
+                            W3CMailList.get_messages_urls(
+                                name=name, url=_period_url
+                            )
+                        )
+                        if nr_msgs > 0:
+                            archive.append(mlist_url)
+                            break
+        else:
+            # collect mailing-list contents
+            for mlist_url in mlist_urls:
+                name = W3CMailList.get_name_from_url(mlist_url)
+                mlist = W3CMailList.from_url(
+                    name=name,
+                    url=mlist_url,
+                    select=select,
+                )
+                if len(mlist) != 0:
+                    if instant_save:
+                        dir_out = CONFIG.mail_path + name
+                        Path(dir_out).mkdir(parents=True, exist_ok=True)
+                        mlist.to_mbox(dir_out=CONFIG.mail_path)
+                        archive.append(mlist.name)
+                    else:
+                        logger.info(f"Recorded the list {mlist.name}.")
+                        archive.append(mlist)
+        return archive
+
+
+def text_for_selector(soup: BeautifulSoup, selector: str):
+    """
+    Filter out header or body field from website and return them as utf-8 string.
+    """
+    results = soup.select(selector)
+    if results:
+        result = results[0].get_text(strip=True)
+    else:
+        result = ""
+        logging.debug("No matching text for selector %s", selector)
+
+    return str(result)
+
+
+def parse_dfn_header(header_text):
+    header_texts = str(header_text).split(":", 1)
+    if len(header_texts) == 2:
+        return header_texts[1]
+    else:
+        logging.debug("Split failed on %s", header_text)
+        return ""
diff --git a/bigbang/utils.py b/bigbang/utils.py
index 91311a8c..384560fb 100644
--- a/bigbang/utils.py
+++ b/bigbang/utils.py
@@ -1,13 +1,31 @@
 """
 Miscellaneous utility functions used in other modules.
 """
-
+import os
+import time
+import logging
 import glob
+import yaml
+from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
+from bs4 import BeautifulSoup
+import requests
 
 import networkx as nx
 import pandas as pd
 
+from config.config import CONFIG
+
+filepath_auth = CONFIG.config_path + "authentication.yaml"
+directory_project = str(Path(os.path.abspath(__file__)).parent.parent)
+logging.basicConfig(
+    filename=directory_project + "/scraping.log",
+    filemode="w",
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+)
+logger = logging.getLogger(__name__)
+
 
 def get_paths_to_files_in_directory(
     directory: str, file_dsc: str = "*"
diff --git a/bigbang/w3crawl.py b/bigbang/w3crawl.py
deleted file mode 100644
index 9b49380d..00000000
--- a/bigbang/w3crawl.py
+++ /dev/null
@@ -1,208 +0,0 @@
-import email
-import email.header
-import email.parser
-import gzip
-import logging
-import mailbox
-import os
-import re
-import time
-import urllib.error
-import urllib.parse
-import urllib.request
-from email.mime.text import MIMEText
-
-import dateutil
-from bs4 import BeautifulSoup
-
-import bigbang.archive as archive
-import bigbang.ingress.mailman as mailman
-
-
-class W3cMailingListArchivesParser(email.parser.Parser):
-    """
-    A subclass of email.parser.Parser that parses the HTML of single-message
-    archive pages generated by W3C's mail archives system, based on Hypermail.
-    """
-
-    parse = None
-    # doesn't implement the file version
-
-    # TODO: ignore spam (has separate error message in w3c archives)
-    def parsestr(self, text, headersonly=False):
-        """
-        Takes the full HTML of a single message page; returns an email Message
-        as an mboxMessage, with appropriate From separator line.
-
-        headersonly is not supported. Not all headers are being parsed yet.
-        """
-        soup = BeautifulSoup(text, "html.parser")
-        body = self._text_for_selector(soup, "#body")
-        msg = MIMEText(body, "plain", "utf-8")
-
-        from_text = self._parse_dfn_header(
-            self._text_for_selector(soup, "#from")
-        )
-        from_name = from_text.split("<")[0].strip()
-        from_address = self._text_for_selector(soup, "#from a")
-
-        from_addr = email.utils.formataddr(
-            (from_name, email.header.Header(from_address).encode())
-        )
-        msg["From"] = from_addr
-
-        subject = self._text_for_selector(soup, "h1")
-        msg["Subject"] = subject
-
-        message_id = self._parse_dfn_header(
-            self._text_for_selector(soup, "#message-id")
-        )
-        msg["Message-ID"] = message_id.strip()
-
-        message_date = self._parse_dfn_header(
-            self._text_for_selector(soup, "#date")
-        )
-        msg["Date"] = message_date.strip()
-
-        message_to = self._parse_dfn_header(
-            self._text_for_selector(soup, "#to")
-        )
-        if message_to:
-            msg["To"] = message_to.strip()
-
-        message_cc = self._parse_dfn_header(
-            self._text_for_selector(soup, "#cc")
-        )
-        if message_cc:
-            msg["Cc"] = message_cc.strip()
-
-        in_reply_to_pattern = re.compile('<!-- inreplyto="(.+?)"')
-        match = in_reply_to_pattern.search(str(text))
-        if match:
-            msg["In-Reply-To"] = "<" + match.groups()[0] + ">"
-
-        mbox_message = mailbox.mboxMessage(msg)
-        mbox_message.set_from(
-            email.header.Header(from_address).encode(),
-            email.utils.parsedate(message_date),
-        )
-
-        return mbox_message
-
-    def _parse_dfn_header(self, header_text):
-        header_texts = str(header_text).split(":", 1)
-        if len(header_texts) == 2:
-            return header_texts[1]
-        else:
-            logging.debug("Split failed on %s", header_text)
-            return ""
-
-    def _text_for_selector(self, soup, selector):
-        results = soup.select(selector)
-        if results:
-            result = results[0].get_text(strip=True)
-        else:
-            result = ""
-            logging.debug("No matching text for selector %s", selector)
-
-        return str(result)
-
-
-def collect_from_url(url, base_arch_dir="archives", notes=None):
-    """
-    Collects W3C mailing list archives from a particular mailing list URL.
-
-    Logs an error and returns False if no messages can be collected.
-    """
-    list_name = mailman.get_list_name(url)
-    logging.info("Getting W3C list archive for %s", list_name)
-
-    try:
-        response = urllib.request.urlopen(url)
-        response_url = response.geturl()
-        html = response.read()
-        soup = BeautifulSoup(html, "html.parser")
-    except urllib.error.HTTPError:
-        logging.exception("Error in loading W3C list archive page for %s", url)
-        return False
-
-    try:
-        time_period_indices = list()
-        rows = soup.select("tbody tr")
-        for row in rows:
-            link = row.select("td:nth-of-type(1) a")[0].get("href")
-            logging.info("Found time period archive page: %s", link)
-            time_period_indices.append(link)
-    except Exception:
-        logging.exception("Error in parsing list archives for %s", url)
-        return False
-
-    # directory for downloaded files
-    arc_dir = archive.archive_directory(base_arch_dir, list_name)
-    mailman.populate_provenance(
-        directory=arc_dir,
-        list_name=list_name,
-        list_url=url,
-        notes=notes,
-    )
-
-    for link in time_period_indices:
-
-        try:
-            link_url = urllib.parse.urljoin(response_url, link)
-            response = urllib.request.urlopen(link_url)
-            html = response.read()
-            soup = BeautifulSoup(html, "html.parser")
-        except urllib.error.HTTPError:
-            logging.exception("Error in loading: %s", link_url)
-            return False
-
-        end_date_string = (
-            soup.select("#end")[0].parent.parent.select("em")[0].get_text()
-        )
-        end_date = dateutil.parser.parse(end_date_string)
-        year_month_mbox = end_date.strftime("%Y-%m") + ".mbox"
-        mbox_path = os.path.join(arc_dir, year_month_mbox)
-
-        # looks like we've already downloaded this timeperiod
-        if os.path.isfile(mbox_path):
-            logging.info("Looks like %s already exists, moving on.", mbox_path)
-            continue
-        logging.info("Downloading messages to archive to %s.", mbox_path)
-
-        message_links = list()
-        messages = list()
-
-        anchors = soup.select("div.messages-list a")
-        for anchor in anchors:
-            if anchor.get("href"):
-                message_url = urllib.parse.urljoin(
-                    link_url, anchor.get("href")
-                )
-                message_links.append(message_url)
-
-        for message_link in message_links:
-            response = urllib.request.urlopen(message_link)
-            html = response.read()
-
-            message = W3cMailingListArchivesParser().parsestr(html)
-            message.add_header("Archived-At", "<" + message_link + ">")
-            messages.append(message)
-            time.sleep(1)  # wait between loading messages, for politeness
-
-        mbox = mailbox.mbox(mbox_path)
-        mbox.lock()
-
-        try:
-            for message in messages:
-                mbox.add(message)
-            mbox.flush()
-        finally:
-            mbox.unlock()
-
-        logging.info("Saved %s", year_month_mbox)
-
-    # assumes all archives were downloaded if no exceptions have been thrown
-    provenance = mailman.access_provenance(arc_dir)
-    provenance["complete"] = True
-    mailman.update_provenance(arc_dir, provenance)


Test patch:
diff --git a/tests/analysis/test_listserv.py b/tests/analysis/test_listserv.py
index 03110634..130731fb 100644
--- a/tests/analysis/test_listserv.py
+++ b/tests/analysis/test_listserv.py
@@ -8,8 +8,8 @@
 import pytest
 import yaml
 
-from bigbang.analysis.listserv import ListservArchive
-from bigbang.analysis.listserv import ListservList
+from bigbang.analysis.listserv import ListservMailListDomain
+from bigbang.analysis.listserv import ListservMailList
 
 from config.config import CONFIG
 
@@ -21,7 +21,7 @@
 
 @pytest.fixture(name="march", scope="module")
 def get_mailingarchive():
-    march = ListservArchive.from_mbox(
+    march = ListservMailListDomain.from_mbox(
         name="3GPP",
         directorypath=CONFIG.test_data_path + "3GPP_mbox/",
         filedsc="3GPP_TSG_*",
@@ -31,22 +31,78 @@ def get_mailingarchive():
 
 @pytest.fixture(name="mlist", scope="module")
 def get_mailinglist():
-    mlist = ListservList.from_mbox(
+    mlist = ListservMailList.from_mbox(
         name="3GPP_TSG_SA_WG4_EVS",
         filepath=CONFIG.test_data_path + "3GPP_mbox/3GPP_TSG_SA_WG4_EVS.mbox",
     )
     return mlist
 
 
-class TestListservList:
+def test_dissection_of_address_header_field():
+    filepath = CONFIG.test_data_path + "address_header_test_file.yaml"
+    with open(filepath, "r") as stream:
+        addresses = yaml.safe_load(stream)
+
+    generator = ListservMailList.iterator_name_localpart_domain(
+        list(addresses.keys())
+    )
+    assert (
+        list(next(generator))
+        == addresses[
+            '"xuxiaodong@chinamobile.com" <xuxiaodong@CHINAMOBILE.COM>'
+        ]
+    )
+    assert (
+        list(next(generator))
+        == addresses["Jacob John <jacobjohn@MOTOROLA.COM>"]
+    )
+    assert (
+        list(next(generator))
+        == addresses['"Wuyuchun (Wu Yuchun, Hisilicon)" <wuyuchun@HUAWEI.COM>']
+    )
+    assert (
+        list(next(generator))
+        == addresses[
+            "xingjinqiang@chinamobile.com xingjinqiang@chinamobile.com"
+        ]
+    )
+    assert (
+        list(next(generator))
+        == addresses["abdul rasheed m d rasheed@motorola.com"]
+    )
+    assert (
+        list(next(generator))
+        == addresses[
+            "fredrik =?utf-8?q?sundstr=c3=b6m?= fredrik.sundstrom@ericsson.com"
+        ]
+    )
+    assert (
+        list(next(generator))
+        == addresses["guozhili guozhili@starpointcomm.com"]
+    )
+    assert (
+        list(next(generator))
+        == addresses["=?utf-8?b?runjvc3lvkdnu6flrr4=?= zhangjibin@ecit.org.cn"]
+    )
+    replyto_sample = addresses[
+        '"Yanyali (Yali)" <yanyali@HUAWEI.COM>, Guchunying <guchunying@HUAWEI.COM>, Ingbert Sigovich <Ingbert.Sigovich@ETSI.ORG>'
+    ]
+    assert list(next(generator)) == replyto_sample[0][1]
+    assert list(next(generator)) == replyto_sample[1][2]
+    assert list(next(generator)) == replyto_sample[2][3]
+
+
+class TestListservMailList:
     def test__to_percentage(self):
         abso = np.array([1, 3])
-        perc = ListservList.to_percentage(abso)
+        perc = ListservMailList.to_percentage(abso)
         np.testing.assert_array_equal(perc, np.array([0.25, 0.75]))
 
     def test__get_name_localpart_domain(self):
         addr = '"gabin frederic" <frederic.gabin@dolby.com>'
-        name, localpart, domain = ListservList.get_name_localpart_domain(addr)
+        name, localpart, domain = ListservMailList.get_name_localpart_domain(
+            addr
+        )
         assert name == "gabin frederic"
         assert localpart == "frederic.gabin"
         assert domain == "dolby.com"
diff --git a/tests/data/address_header_test_file.yaml b/tests/data/address_header_test_file.yaml
index acf5dc8d..44e236a8 100644
--- a/tests/data/address_header_test_file.yaml
+++ b/tests/data/address_header_test_file.yaml
@@ -1,40 +1,49 @@
 '"xuxiaodong@chinamobile.com" <xuxiaodong@CHINAMOBILE.COM>':
-  name: 'xuxiaodong@chinamobile.com'
-  localpart: 'xuxiaodong'
-  domain: 'CHINAMOBILE.COM'
+  - 'xuxiaodong@chinamobile.com'
+  - 'xuxiaodong'
+  - 'chinamobile.com'
 'Jacob John <jacobjohn@MOTOROLA.COM>':
-  name: 'Jacob John'
-  localpart: 'jacobjohn'
-  domain: 'MOTOROLA.COM'
+  - 'jacob john'
+  - 'jacobjohn'
+  - 'motorola.com'
 '"Wuyuchun (Wu Yuchun, Hisilicon)" <wuyuchun@HUAWEI.COM>':
-  name: 'Wuyuchun (Wu Yuchun, Hisilicon)'
-  localpart: 'wuyuchun'
-  domain: 'HUAWEI.COM'
+  - 'wuyuchun (wu yuchun, hisilicon)'
+  - 'wuyuchun'
+  - 'huawei.com'
 'xingjinqiang@chinamobile.com xingjinqiang@chinamobile.com':
-  name: null
-  localpart: 'xingjinqiang'
-  domain: 'chinamobile.com'
+  - null
+  - 'xingjinqiang'
+  - 'chinamobile.com'
 'abdul rasheed m d rasheed@motorola.com':
-  name: 'abdul rasheed m d'
-  localpart: 'rasheed'
-  domain: 'motorola.com'
+  - 'abdul rasheed m d'
+  - 'rasheed'
+  - 'motorola.com'
 'fredrik =?utf-8?q?sundstr=c3=b6m?= fredrik.sundstrom@ericsson.com':
-  name: 'fredrik =?utf-8?q?sundstr=c3=b6m?='
-  localpart: 'fredrik.sundstrom'
-  domain: 'ericsson.com'
+  - 'fredrik =?utf-8?q?sundstr=c3=b6m?='
+  - 'fredrik.sundstrom'
+  - 'ericsson.com'
 'guozhili guozhili@starpointcomm.com':
-  name: 'guozhili'
-  localpart: 'guozhili'
-  domain: 'starpointcomm.com'
+  - 'guozhili'
+  - 'guozhili'
+  - 'starpointcomm.com'
 '=?utf-8?b?runjvc3lvkdnu6flrr4=?= zhangjibin@ecit.org.cn':
-  name: '=?utf-8?b?runjvc3lvkdnu6flrr4=?='
-  localpart: 'zhangjibin'
-  domain: 'ecit.org.cn'
+  - '=?utf-8?b?runjvc3lvkdnu6flrr4=?='
+  - 'zhangjibin'
+  - 'ecit.org.cn'
 'Xueming Pan <<a href="/scripts/wa.exe?LOGON=A2%3D3GPP_TSG_RAN%3B82086c35.2012B%26S%3D&amp" target="_parent">[log in to unmask]</a>> <a href="/scripts/wa.exe?LOGON=A2%3D3GPP_TSG_RAN%3B82086c35.2012B%26S%3D&amp" target="_parent" /a':
-  name: null
-  localpart: null
-  domain: null
+  - null
+  - null
+  - null
 '"Yanyali (Yali)" <yanyali@HUAWEI.COM>, Guchunying <guchunying@HUAWEI.COM>, Ingbert Sigovich <Ingbert.Sigovich@ETSI.ORG>':
-  name: ['Yanyali (Yali)', 'Guchunying', 'Ingbert Sigovich']
-  localpart: ['yanyali', 'guchunying', 'Ingbert.Sigovich']
-  domain: ['HUAWEI.COM', 'HUAWEI.COM', 'ETSI.ORG']
+  - 1:
+    - 'yanyali@huawei.com'
+    - 'yanyali'
+    - 'huawei.com'
+  - 2:
+    - guchunying@huawei.com
+    - 'guchunying'
+    - 'huawei.com'
+  - 3:
+    - 'ingbert.sigovich@etsi.org'
+    - 'ingbert.sigovich'
+    - 'etsi.org'
diff --git a/tests/webscraping/__init__.py b/tests/ingress/__init__.py
similarity index 100%
rename from tests/webscraping/__init__.py
rename to tests/ingress/__init__.py
diff --git a/tests/webscraping/test_listserv.py b/tests/ingress/test_listserv.py
similarity index 74%
rename from tests/webscraping/test_listserv.py
rename to tests/ingress/test_listserv.py
index 3a2b5e43..7a3c0a6f 100644
--- a/tests/webscraping/test_listserv.py
+++ b/tests/ingress/test_listserv.py
@@ -7,18 +7,18 @@
 import yaml
 
 import bigbang
-from bigbang import listserv
-from bigbang.listserv import (
-    ListservArchive,
-    ListservList,
+from bigbang.ingress import (
     ListservMessageParser,
+    ListservMailList,
+    ListservMailListDomain,
 )
+from bigbang.ingress.utils import get_login_from_terminal
 from config.config import CONFIG
 
 dir_temp = tempfile.gettempdir()
-url_archive = "https://listserv.ieee.org/cgi-bin/wa?"
-url_list = url_archive + "A0=IEEE-TEST"
-url_message = url_archive + "A2=ind1511d&L=IEEE-TEST&P=67"
+url_mlistdom = "https://listserv.ieee.org/cgi-bin/wa?"
+url_list = url_mlistdom + "A0=IEEE-TEST"
+url_message = url_mlistdom + "A2=ind1511d&L=IEEE-TEST&P=67"
 file_temp_mbox = dir_temp + "/listserv.mbox"
 file_auth = CONFIG.config_path + "authentication.yaml"
 auth_key_mock = {"username": "bla", "password": "bla"}
@@ -34,6 +34,8 @@ def test__from_url_with_login(self):
             auth_key = yaml.safe_load(stream)
         msg_parser = ListservMessageParser(
             website=True,
+            url_login="https://listserv.ieee.org/cgi-bin/wa?LOGON",
+            url_pref="https://listserv.ieee.org/cgi-bin/wa?PREF",
             login=auth_key,
         )
         msg = msg_parser.from_url(
@@ -42,13 +44,15 @@ def test__from_url_with_login(self):
             fields="total",
         )
         print(list(msg.keys()))
-        assert msg["from"] == 'iceeng-10'
+        assert msg["from"] == "iceeng-10"
         assert msg["in-reply-to"] is None
 
     @pytest.fixture(name="msg", scope="module")
     def get_message(self):
         msg_parser = ListservMessageParser(
             website=True,
+            url_login="https://listserv.ieee.org/cgi-bin/wa?LOGON",
+            url_pref="https://listserv.ieee.org/cgi-bin/wa?PREF",
             login=auth_key_mock,
         )
         msg = msg_parser.from_url(
@@ -79,6 +83,8 @@ def test__message_content(self, msg):
     def test__only_header_from_url(self):
         msg_parser = ListservMessageParser(
             website=True,
+            url_login="https://listserv.ieee.org/cgi-bin/wa?LOGON",
+            url_pref="https://listserv.ieee.org/cgi-bin/wa?PREF",
             login=auth_key_mock,
         )
         msg = msg_parser.from_url(
@@ -91,6 +97,8 @@ def test__only_header_from_url(self):
     def test__only_body_from_url(self):
         msg_parser = ListservMessageParser(
             website=True,
+            url_login="https://listserv.ieee.org/cgi-bin/wa?LOGON",
+            url_pref="https://listserv.ieee.org/cgi-bin/wa?PREF",
             login=auth_key_mock,
         )
         msg = msg_parser.from_url(
@@ -98,7 +106,7 @@ def test__only_body_from_url(self):
             url=url_message,
             fields="body",
         )
-        assert str(msg["subject"]) == 'None'
+        assert str(msg["subject"]) == "None"
 
     def test__to_dict(self, msg):
         dic = ListservMessageParser.to_dict(msg)
@@ -114,8 +122,7 @@ def test__to_mbox(self, msg):
         Path(file_temp_mbox).unlink()
 
 
-class TestListservList:
-
+class TestListservMailList:
     @pytest.mark.skipif(
         not os.path.isfile(file_auth),
         reason="Key to log into LISTSERV could not be found",
@@ -123,7 +130,7 @@ class TestListservList:
     def test__from_url_with_login(self):
         with open(file_auth, "r") as stream:
             auth_key = yaml.safe_load(stream)
-        mlist = ListservList.from_url(
+        mlist = ListservMailList.from_url(
             name="IEEE-TEST",
             url=url_list,
             select={
@@ -132,14 +139,16 @@ def test__from_url_with_login(self):
                 "weeks": 4,
                 "fields": "header",
             },
+            url_login="https://listserv.ieee.org/cgi-bin/wa?LOGON",
+            url_pref="https://listserv.ieee.org/cgi-bin/wa?PREF",
             login=auth_key,
         )
-        assert mlist.messages[0]["from"] == 'iceeng-10'
+        assert mlist.messages[0]["from"] == "iceeng-10"
         assert mlist.messages[0]["to"] is None
-    
+
     @pytest.fixture(name="mlist", scope="module")
-    def get_mailinglist_from_url(self):
-        mlist = ListservList.from_url(
+    def get_maillist_from_url(self):
+        mlist = ListservMailList.from_url(
             name="IEEE-TEST",
             url=url_list,
             select={
@@ -149,13 +158,13 @@ def get_mailinglist_from_url(self):
             login=auth_key_mock,
         )
         return mlist
-    
-    def test__get_mailinglist_from_messages(self):
+
+    def test__get_maillist_from_messages(self):
         msgs_urls = [
             "https://listserv.ieee.org/cgi-bin/wa?A2=IEEE-TEST;a57724f6.2105a",
             "https://listserv.ieee.org/cgi-bin/wa?A2=IEEE-TEST;fc0a9fdd.2105b",
         ]
-        mlist = ListservList.from_messages(
+        mlist = ListservMailList.from_messages(
             name="IEEE-TEST",
             url=url_list,
             messages=msgs_urls,
@@ -191,42 +200,44 @@ def test__to_mbox(self, mlist):
         Path(file_temp_mbox).unlink()
 
 
-class TestListservArchive:
-    @pytest.fixture(name="arch", scope="session")
-    def get_mailarchive(self):
-        arch = ListservArchive.from_url(
+class TestListservMailListDomain:
+    @pytest.fixture(name="mlistdom", scope="session")
+    def get_maillistdomain(self):
+        mlistdom = ListservMailListDomain.from_url(
             name="IEEE",
-            url_root=url_archive,
-            url_home=url_archive + "HOME",
+            url_root=url_mlistdom,
+            url_home=url_mlistdom + "HOME",
             select={
                 "years": 2015,
                 "months": "November",
                 "weeks": 4,
                 "fields": "header",
             },
+            url_login="https://listserv.ieee.org/cgi-bin/wa?LOGON",
+            url_pref="https://listserv.ieee.org/cgi-bin/wa?PREF",
             login=auth_key_mock,
             instant_save=False,
             only_mlist_urls=False,
         )
-        return arch
+        return mlistdom
 
-    def test__archive_content(self, arch):
-        assert arch.name == "IEEE"
-        assert arch.url == url_archive
-        assert len(arch) == 1
-        assert len(arch.lists[0]) == 1
+    def test__maillistdomain_content(self, mlistdom):
+        assert mlistdom.name == "IEEE"
+        assert mlistdom.url == url_mlistdom
+        assert len(mlistdom) == 1
+        assert len(mlistdom.lists[0]) == 1
         assert (
-            arch.lists[0].messages[0]["subject"]
+            mlistdom.lists[0].messages[0]["subject"]
             == '"10th International Conference on Electrical Engineering (ICEENG\'10)"'
         )
 
-    def test__to_dict(self, arch):
-        dic = arch.to_dict()
+    def test__to_dict(self, mlistdom):
+        dic = mlistdom.to_dict()
         assert len(list(dic.keys())) == 8
         assert len(dic[list(dic.keys())[0]]) == 1
 
-    def test__to_mbox(self, arch):
-        arch.to_mbox(dir_temp)
+    def test__to_mbox(self, mlistdom):
+        mlistdom.to_mbox(dir_temp)
         file_dic = {
             f"{dir_temp}/IEEE-TEST.mbox": 14,
         }
@@ -240,11 +251,11 @@ def test__to_mbox(self, arch):
             Path(filepath).unlink()
 
 
-@mock.patch("bigbang.listserv.ask_for_input", return_value="check")
+@mock.patch("bigbang.ingress.utils.ask_for_input", return_value="check")
 def test__get_login_from_terminal(input):
     """test if login keys will be documented"""
     file_auth = dir_temp + "/authentication.yaml"
-    _, _ = listserv.get_login_from_terminal(
+    _, _ = get_login_from_terminal(
         username=None, password=None, file_auth=file_auth
     )
     f = open(file_auth, "r")
diff --git a/tests/ingress/test_w3c.py b/tests/ingress/test_w3c.py
new file mode 100644
index 00000000..aeb3d29a
--- /dev/null
+++ b/tests/ingress/test_w3c.py
@@ -0,0 +1,211 @@
+import os
+import tempfile
+from pathlib import Path
+from unittest import mock
+
+import pytest
+import yaml
+
+import bigbang
+from bigbang.ingress import (
+    W3CMessageParser,
+    W3CMailList,
+    W3CMailListDomain,
+)
+from config.config import CONFIG
+
+dir_temp = tempfile.gettempdir()
+url_mlistdom = "https://lists.w3.org/Archives/Public/"
+url_list = url_mlistdom + "public-test2/"
+url_message = url_mlistdom + "public-test2/2012Sep/0000.html"
+file_temp_mbox = dir_temp + "/w3c.mbox"
+
+
+class TestW3CMessageParser:
+    @pytest.fixture(name="msg", scope="module")
+    def get_message(self):
+        msg_parser = W3CMessageParser(
+            website=True,
+        )
+        msg = msg_parser.from_url(
+            list_name="public-test2",
+            url=url_message,
+            fields="total",
+        )
+        return msg
+
+    def test__message_content(self, msg):
+        lines = msg.get_payload()  # .split("C")[1].split("=")[0]
+        assert lines.split("\n")[0] == "Dear Community Group participant,"
+        assert (
+            msg["Subject"]
+            == "Survey on Community and Business Group Experience [reminder]"
+        )
+        assert msg["From"] == "Coralie Mercier <coralie@w3.org>"
+        assert (
+            msg["To"]
+            == '"team-community-process@w3.org" <team-community-process@w3.org>'
+        )
+        assert msg["Date"] == "Fri, 07 Sep 2012 00:50:51 +0200"
+        assert msg["Content-Type"] == 'text/plain; charset="utf-8"'
+
+    def test__only_header_from_url(self):
+        msg_parser = W3CMessageParser(website=True)
+        msg = msg_parser.from_url(
+            list_name="public-test2",
+            url=url_message,
+            fields="header",
+        )
+        assert msg.get_payload() is None
+
+    def test__only_body_from_url(self):
+        msg_parser = W3CMessageParser(website=True)
+        msg = msg_parser.from_url(
+            list_name="public-test2",
+            url=url_message,
+            fields="body",
+        )
+        assert str(msg["Subject"]) == "None"
+
+    def test__to_dict(self, msg):
+        dic = W3CMessageParser.to_dict(msg)
+        assert len(list(dic.keys())) == 11
+
+    def test__to_mbox(self, msg):
+        W3CMessageParser.to_mbox(msg, filepath=file_temp_mbox)
+        f = open(file_temp_mbox, "r")
+        lines = f.readlines()
+        assert len(lines) == 37
+        assert lines[1] == 'Content-Type: text/plain; charset="utf-8"\n'
+        f.close()
+        Path(file_temp_mbox).unlink()
+
+
+class TestW3CMailList:
+    @pytest.fixture(name="mlist", scope="module")
+    def get_mailinglist_from_url(self):
+        mlist = W3CMailList.from_url(
+            name="public-test2",
+            url=url_list,
+            select={
+                "years": 2019,
+                "fields": "header",
+            },
+        )
+        return mlist
+
+    def test__get_mailinglist_from_messages(self):
+        msgs_urls = [
+            "https://lists.w3.org/Archives/Public/public-test2/2019Apr/0000.html",
+            "https://lists.w3.org/Archives/Public/public-test2/2019Mar/0000.html",
+        ]
+        mlist = W3CMailList.from_messages(
+            name="public-test2",
+            url=url_list,
+            messages=msgs_urls,
+        )
+        assert len(mlist.messages) == 2
+
+    def test__mailinglist_content(self, mlist):
+        assert mlist.name == "public-test2"
+        assert mlist.source == url_list
+        assert len(mlist) == 2
+        assert (
+            mlist.messages[0]["Subject"]
+            == "Re: W3C TPAC 2019 - Will your Community Group meet in Fukuoka?"
+        )
+
+    def test__to_dict(self, mlist):
+        dic = mlist.to_dict()
+        assert len(list(dic.keys())) == 9
+        assert len(dic[list(dic.keys())[0]]) == 2
+
+    def test__to_mbox(self, mlist):
+        mlist.to_mbox(dir_temp, filename=mlist.name)
+        file_temp_mbox = f"{dir_temp}/{mlist.name}.mbox"
+        f = open(file_temp_mbox, "r")
+        lines = f.readlines()
+        assert len(lines) == 31
+        assert (
+            lines[5]
+            == "Subject: Re: W3C TPAC 2019 - Will your Community Group meet in Fukuoka?\n"
+        )
+        f.close()
+        Path(file_temp_mbox).unlink()
+
+
+class TestW3CMailListDomain:
+    # def test__get_only_mlist_urls(self):
+    #    arch = W3CMailListDomain.from_url(
+    #        name="W3C",
+    #        url_root=url_mlistdom,
+    #        select={
+    #            "years": 2015,
+    #            "months": "November",
+    #        },
+    #        instant_save=False,
+    #        only_mlist_urls=True,
+    #    )
+    #    return arch
+
+    @pytest.fixture(name="mlistdom", scope="session")
+    def get_maillistdomain(self):
+        mlistdom = W3CMailListDomain.from_mailing_lists(
+            name="W3C",
+            url_root=url_mlistdom,
+            url_mailing_lists=[
+                "https://lists.w3.org/Archives/Public/public-ixml/",
+                "https://lists.w3.org/Archives/Public/public-traffic/",
+                "https://lists.w3.org/Archives/Public/wai-site-comments/",
+            ],
+            select={
+                "years": 2015,
+                "months": "November",
+                "fields": "header",
+            },
+            instant_save=False,
+            only_mlist_urls=False,
+        )
+        return mlistdom
+
+    def test__from_mbox(self):
+        mlistdom = W3CMailListDomain.from_mbox(
+            name="3GPP",
+            directorypath=CONFIG.test_data_path + "3GPP_mbox/",
+            filedsc="3GPP_TSG_*.mbox",
+        )
+        assert len(mlistdom) == 2
+        names = [mlist.name for mlist in mlistdom.lists]
+        index = names.index("3GPP_TSG_SA_WG4_EVS")
+        assert len(mlistdom.lists[index]) == 50
+        index = names.index("3GPP_TSG_RAN_WG4_NR-MIMO-OTA")
+        assert len(mlistdom.lists[index]) == 59
+
+    def test__maillistdomain_content(self, mlistdom):
+        assert mlistdom.name == "W3C"
+        assert mlistdom.url == url_mlistdom
+        assert len(mlistdom) == 1
+        assert len(mlistdom.lists[0]) == 2
+        assert (
+            mlistdom.lists[0].messages[0]["Subject"]
+            == "Re: Web Accessibility Issue"
+        )
+
+    def test__to_dict(self, mlistdom):
+        dic = mlistdom.to_dict()
+        assert len(list(dic.keys())) == 10
+        assert len(dic[list(dic.keys())[0]]) == 2
+
+    def test__to_mbox(self, mlistdom):
+        mlistdom.to_mbox(dir_temp)
+        file_dic = {
+            f"{dir_temp}/wai-site-comments.mbox": 25,
+        }
+        for filepath, line_nr in file_dic.items():
+            assert Path(filepath).is_file()
+            f = open(filepath, "r")
+            lines = f.readlines()
+            lines = [ll for ll in lines if len(ll) > 1]
+            assert line_nr == len(lines)
+            f.close()
+            Path(filepath).unlink()
diff --git a/tests/unit/test_bigbang.py b/tests/unit/test_bigbang.py
index 80b9b435..4582fe84 100644
--- a/tests/unit/test_bigbang.py
+++ b/tests/unit/test_bigbang.py
@@ -16,7 +16,6 @@
 import bigbang.parse as parse
 import bigbang.analysis.process as process
 import bigbang.utils as utils
-import bigbang.w3crawl as w3crawl
 from bigbang.analysis import repo_loader
 from config.config import CONFIG
 
@@ -183,120 +182,3 @@ def test_labeled_blockmodel(self):
             list(bg.edges()) == [("A", "B")],
             msg="Incorrected edges in labeled blockmodel",
         )
-
-
-class TestW3crawl(unittest.TestCase):
-    def setUp(self):
-        try:
-            os.mkdir(TEMP_DIR)
-        except FileExistsError:
-            shutil.rmtree(TEMP_DIR)
-            os.mkdir(TEMP_DIR)
-
-    def tearDown(self):
-        shutil.rmtree(TEMP_DIR)
-
-    def test_w3c_archive_message_parsing(self):
-        test_html_path = os.path.join(
-            CONFIG.test_data_path, "w3crawl-test-message.html"
-        )
-        f = open(test_html_path, "r")
-        html = f.read()
-        f.close()
-        message = w3crawl.W3cMailingListArchivesParser().parsestr(html)
-
-        assert (
-            len(message.get_from()) > 0
-        ), "message doesn't have a From address"
-        assert message.get_from().startswith(
-            "npdoty@ischool.berkeley.edu"
-        ), "incorrect From address parsed"
-        assert "Subject:" in str(
-            message
-        ), "message does not have Subject header"
-        assert "Message-ID:" in str(
-            message
-        ), "message does not have message id"
-        assert "Date:" in str(message), "message does not have Date header"
-        assert "In-Reply-To:" not in str(
-            message
-        ), "this message shouldn't have an in-reply-to"
-
-    def test_non_ascii_address_parsing(self):
-        test_html_path = os.path.join(
-            CONFIG.test_data_path, "w3crawl-nonascii.html"
-        )
-        f = open(test_html_path, "r")
-        html = f.read()
-        f.close()
-        message = w3crawl.W3cMailingListArchivesParser().parsestr(html)
-
-        assert (
-            len(message.get_from()) > 0
-        ), "message doesn't have a From address"
-
-        mbox = mailbox.mbox(os.path.join(TEMP_DIR, "test.mbox"))
-        mbox.add(
-            message
-        )  # make sure it doesn't crash on adding to an mbox either
-
-    def test_w3c_archive_message_headers(self):
-        test_html_path = os.path.join(
-            CONFIG.test_data_path, "w3crawl-test-message-to-cc.html"
-        )
-        f = open(test_html_path, "r")
-        html = f.read()
-        f.close()
-        message = w3crawl.W3cMailingListArchivesParser().parsestr(html)
-
-        assert "Subject:" in str(
-            message
-        ), "message does not have Subject header"
-        assert "Message-ID:" in str(
-            message
-        ), "message does not have message id"
-        assert "Cc" in message, "message does not have expected CC header"
-        assert (
-            "," in message["To"]
-        ), "message does not have a To header with multiple addresses"
-        assert (
-            "In-Reply-To" in message
-        ), "this message should have an in-reply-to header"
-        assert (
-            message["In-Reply-To"]
-            == "<CABQTWrn2fqvee6qg2VTcDA5rR8QihQy7qjycDGBM2xafwGyRmQ&#64;mail.gmail.com>"
-        )
-
-    def test_w3c_collect_from_url(self):
-        test_archive_homepage_path = os.path.join(
-            CONFIG.test_data_path, "w3c-archive-dir", "index"
-        )
-        test_archive_homepage_url = pathlib.Path(
-            test_archive_homepage_path
-        ).as_uri()
-
-        w3crawl.collect_from_url(
-            test_archive_homepage_url,
-            base_arch_dir=TEMP_DIR,
-            notes="w3crawl test",
-        )
-
-        assert os.path.isfile(
-            os.path.join(TEMP_DIR, "index", "2020-05.mbox")
-        ), "mbox file was not created"
-        assert os.path.isfile(
-            os.path.join(TEMP_DIR, "index", mailman.PROVENANCE_FILENAME)
-        ), "provenance file was not created"
-
-        provenance = mailman.access_provenance(os.path.join(TEMP_DIR, "index"))
-        assert provenance["notes"] == "w3crawl test"
-        assert (
-            provenance["complete"] is True
-        ), "provenance not marked as a complete crawl"
-
-        mbox = mailbox.mbox(os.path.join(TEMP_DIR, "index", "2020-05.mbox"))
-        for message in mbox:
-            assert "Archived-At" in message
-            assert re.match(
-                "<.+>", message["Archived-At"]
-            ), "Archived-At header format not as expected"
diff --git a/tests/unit/test_listserv.py b/tests/unit/test_listserv.py
index 55f1f707..1fb589fb 100644
--- a/tests/unit/test_listserv.py
+++ b/tests/unit/test_listserv.py
@@ -8,11 +8,10 @@
 import yaml
 
 import bigbang
-from bigbang import listserv
-from bigbang.listserv import (
-    ListservArchive,
-    ListservList,
+from bigbang.ingress.listserv import (
     ListservMessageParser,
+    ListservMailList,
+    ListservMailListDomain,
 )
 from config.config import CONFIG
 
@@ -23,8 +22,8 @@
 
 
 @pytest.fixture(name="mlist", scope="module")
-def get_mailinglist():
-    mlist = ListservList.from_listserv_directories(
+def get_maillist():
+    mlist = ListservMailList.from_listserv_directories(
         name="3GPP_TSG_SA_ITUT_AHG",
         directorypaths=[CONFIG.test_data_path + "3GPP/3GPP_TSG_SA_ITUT_AHG/"],
     )
@@ -83,10 +82,10 @@ def test__to_mbox(self, msg_parser, msg):
         Path(file_temp_mbox).unlink()
 
 
-class TestListservList:
+class TestListservMailList:
     def test__from_mbox(self):
         mlist_name = "3GPP_TSG_SA_WG4_EVS"
-        mlist = ListservList.from_mbox(
+        mlist = ListservMailList.from_mbox(
             name=mlist_name,
             filepath=CONFIG.test_data_path + f"3GPP_mbox/{mlist_name}.mbox",
         )
@@ -101,7 +100,7 @@ def test__from_listserv_files(self):
             CONFIG.test_data_path
             + "3GPP/3GPP_TSG_SA_ITUT_AHG/3GPP_TSG_SA_ITUT_AHG.LOG1703B"
         )
-        mlist = ListservList.from_listserv_files(
+        mlist = ListservMailList.from_listserv_files(
             name="3GPP_TSG_SA_ITUT_AHG",
             filepaths=[filepath],
         )
@@ -153,9 +152,9 @@ def test__missing_date_in_message(self, mlist):
         Path(file_temp_mbox).unlink()
 
 
-class TestListservArchive:
+class TestListservMailListDomain:
     def test__from_mbox(self):
-        march = ListservArchive.from_mbox(
+        march = ListservMailListDomain.from_mbox(
             name="3GPP_mbox_test",
             directorypath=CONFIG.test_data_path + "3GPP_mbox/",
         )
@@ -170,7 +169,7 @@ def test__from_mbox(self):
 
     @pytest.fixture(name="arch", scope="session")
     def get_mailarchive(self):
-        arch = ListservArchive.from_listserv_directory(
+        arch = ListservMailListDomain.from_listserv_directory(
             name="3GPP",
             directorypath=CONFIG.test_data_path + "3GPP/",
         )
@@ -222,17 +221,3 @@ def test__to_mbox(self, arch):
             assert line_nr < len(lines)
             f.close()
             Path(filepath).unlink()
-
-
-@mock.patch("bigbang.listserv.ask_for_input", return_value="check")
-def test__get_login_from_terminal(input):
-    """test if login keys will be documented"""
-    file_auth = dir_temp + "/authentication.yaml"
-    _, _ = listserv.get_login_from_terminal(
-        username=None, password=None, file_auth=file_auth
-    )
-    f = open(file_auth, "r")
-    lines = f.readlines()
-    assert lines[0].strip("\n") == "username: 'check'"
-    assert lines[1].strip("\n") == "password: 'check'"
-    os.remove(file_auth)