Improve code and add more example specific text

app.py

@@ -1,8 +1,10 @@
 import random
 from typing import AnyStr
+# import tensorflow_hub as hub
 
 import itertools
 import streamlit as st
+
 import torch.nn.parameter
 from bs4 import BeautifulSoup
 import numpy as np
@@ -15,6 +17,7 @@ from validators import ValidationFailure
 from custom_renderer import render_sentence_custom
 from flair.data import Sentence
 from flair.models import SequenceTagger
+from sentence_transformers import SentenceTransformer
 
 import spacy
 from spacy import displacy
@@ -25,30 +28,8 @@ from transformers import pipeline
 import os
 from transformers_interpret import SequenceClassificationExplainer
 
-# Map model names to URLs
-model_names_to_URLs = {
-    'ml6team/distilbert-base-dutch-cased-toxic-comments':
-        'https://huggingface.co/ml6team/distilbert-base-dutch-cased-toxic-comments',
-    'ml6team/robbert-dutch-base-toxic-comments':
-        'https://huggingface.co/ml6team/robbert-dutch-base-toxic-comments',
-}
-
-about_page_markdown = f"""# 🤬 Dutch Toxic Comment Detection Space
-
-Made by [ML6](https://ml6.eu/).
-
-Token attribution is performed using [transformers-interpret](https://github.com/cdpierse/transformers-interpret).
-"""
-
-regular_emojis = [
-    '😐', '🙂', '👶', '😇',
-]
-undecided_emojis = [
-    '🤨', '🧐', '🥸', '🥴', '🤷',
-]
-potty_mouth_emojis = [
-    '🤐', '👿', '😡', '🤬', '☠️', '☣️', '☢️',
-]
+# USE_model = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
+sentence_embedding_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
 
 # Page setup
 st.set_page_config(
@@ -64,58 +45,6 @@ st.set_page_config(
 )
 
 
-# Model setup
-@st.cache(allow_output_mutation=True,
-          suppress_st_warning=True,
-          show_spinner=False)
-def load_pipeline(model_name):
-    with st.spinner('Loading model (this might take a while)...'):
-        toxicity_pipeline = pipeline(
-            'text-classification',
-            model=model_name,
-            tokenizer=model_name)
-        cls_explainer = SequenceClassificationExplainer(
-            toxicity_pipeline.model,
-            toxicity_pipeline.tokenizer)
-    return toxicity_pipeline, cls_explainer
-
-
-# Auxiliary functions
-def format_explainer_html(html_string):
-    """Extract tokens with attribution-based background color."""
-    inside_token_prefix = '##'
-    soup = BeautifulSoup(html_string, 'html.parser')
-    p = soup.new_tag('p',
-                     attrs={'style': 'color: black; background-color: white;'})
-    # Select token elements and remove model specific tokens
-    current_word = None
-    for token in soup.find_all('td')[-1].find_all('mark')[1:-1]:
-        text = token.font.text.strip()
-        if text.startswith(inside_token_prefix):
-            text = text[len(inside_token_prefix):]
-        else:
-            # Create a new span for each word (sequence of sub-tokens)
-            if current_word is not None:
-                p.append(current_word)
-                p.append(' ')
-            current_word = soup.new_tag('span')
-        token.string = text
-        token.attrs['style'] = f"{token.attrs['style']}; padding: 0.2em 0em;"
-        current_word.append(token)
-
-    # Add last word
-    p.append(current_word)
-
-    # Add left and right-padding to each word
-    for span in p.find_all('span'):
-        span.find_all('mark')[0].attrs['style'] = (
-            f"{span.find_all('mark')[0].attrs['style']}; padding-left: 0.2em;")
-        span.find_all('mark')[-1].attrs['style'] = (
-            f"{span.find_all('mark')[-1].attrs['style']}; padding-right: 0.2em;")
-
-    return p
-
-
 def list_all_article_names() -> list:
     filenames = []
     for file in sorted(os.listdir('./sample-articles/')):
@@ -148,32 +77,6 @@ def fetch_dependency_specific_contents(filename: str) -> AnyStr:
     return data
 
 
-def classify_comment(comment, selected_model):
-    """Classify the given comment and augment with additional information."""
-    toxicity_pipeline, cls_explainer = load_pipeline(selected_model)
-    result = toxicity_pipeline(comment)[0]
-    result['model_name'] = selected_model
-
-    # Add explanation
-    result['word_attribution'] = cls_explainer(comment, class_name="non-toxic")
-    result['visualitsation_html'] = cls_explainer.visualize()._repr_html_()
-    result['tokens_with_background'] = format_explainer_html(
-        result['visualitsation_html'])
-
-    # Choose emoji reaction
-    label, score = result['label'], result['score']
-    if label == 'toxic' and score > 0.1:
-        emoji = random.choice(potty_mouth_emojis)
-    elif label in ['non_toxic', 'non-toxic'] and score > 0.1:
-        emoji = random.choice(regular_emojis)
-    else:
-        emoji = random.choice(undecided_emojis)
-    result.update({'text': comment, 'emoji': emoji})
-
-    # Add result to session
-    st.session_state.results.append(result)
-
-
 def display_summary(article_name: str):
     summary_content = fetch_summary_contents(article_name)
     st.session_state.summary_output = summary_content
@@ -244,6 +147,10 @@ def get_and_compare_entities(article_name: str):
         # TODO: currently substring matching but probably should do embedding method or idk?
         if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):
             matched_entities.append(entity)
+        elif any(
+                np.inner(sentence_embedding_model.encode(entity), sentence_embedding_model.encode(art_entity)) > 0.9 for
+                art_entity in entities_article):
+            matched_entities.append(entity)
         else:
             unmatched_entities.append(entity)
     return matched_entities, unmatched_entities
@@ -343,26 +250,27 @@ st.title('Summarization fact checker')
 
 # INTRODUCTION
 st.header("Introduction")
-st.markdown("""Recent work using transformers on large text corpora has shown great succes when fine-tuned on several 
-different downstream NLP tasks. One such task is that of text summarization. The goal of text summarization is to 
-generate concise and accurate summaries from input document(s). There are 2 types of summarization: extractive and 
-abstractive. **Exstractive summarization** merely copies informative fragments from the input, whereas **abstractive 
-summarization** may generate novel words. A good abstractive summary should cover principal information in the input 
-and has to be linguistically fluent. This blogpost will focus on this more difficult task of abstractive summary 
-generation.""")
+st.markdown("""Recent work using transformers on large text corpora has shown great success when fine-tuned on 
+several different downstream NLP tasks. One such task is that of text summarization. The goal of text summarization 
+is to generate concise and accurate summaries from input document(s). There are 2 types of summarization: extractive 
+and abstractive. **Extractive summarization** merely copies informative fragments from the input, 
+whereas **abstractive summarization** may generate novel words. A good abstractive summary should cover principal 
+information in the input and has to be linguistically fluent. This blogpost will focus on this more difficult task of 
+abstractive summary generation.""")
 
 st.markdown("""To generate summaries we will use the [PEGASUS] (https://huggingface.co/google/pegasus-cnn_dailymail) 
-model, producing abstractive summaries from large articles. These summaries often still contain sentences with 
-different kinds of errors. Rather than improving the core model, we will look at possible post-processing steps to 
-improve the generated summaries by detecting such possible errors. By comparing contents of the summary with the 
-source text, we can create some sort of factualness metric, indicating the trustworthiness of the generated 
-summary.""")
+model, producing abstractive summaries from large articles. These summaries often contain sentences with different 
+kinds of errors. Rather than improving the core model, we will look into possible post-processing steps to improve 
+the generated summaries. By comparing contents of the summary with the source text, we come up with a factualness 
+metric, indicating the trustworthiness of the generated summary. Throughout this blog, we will also explain the 
+results for some methods on specific examples. These text blocks will be indicated and they change according to the 
+currently selected article.""")
 
 # GENERATING SUMMARIES PART
 st.header("Generating summaries")
 st.markdown("Let’s start by selecting an article text for which we want to generate a summary, or you can provide "
             "text yourself. Note that it’s suggested to provide a sufficiently large text, as otherwise the summary "
-            "generated might not be optimal to start from.")
+            "generated from it might not be optimal, leading to suboptimal performance of the post-processing steps.")
 
 # TODO: NEED TO CHECK ARTICLE TEXT INSTEAD OF ARTICLE NAME ALSO FREE INPUT OPTION
 selected_article = st.selectbox('Select an article or provide your own:',
@@ -374,12 +282,11 @@ article_text = st.text_area(
     height=150
 )
 
-st.markdown("Below you can find the generated summary for the article. The summaries of the example articles "
-            "vary in quality, but are chosen as such. Based on some common errors, we will discuss possible "
-            "methods to improve or rank the summaries in the following paragraphs. The idea is that in "
-            "production, you could generate a set of summaries for the same article, with different "
-            "parameters (or even different models). By using post-processing methods and metrics, "
-            "we can detect some errors in summaries, and choose the best one to actually use.")
+st.markdown("Below you can find the generated summary for the article. Based on empirical research, we will discuss "
+            "two main methods that detect some common errors. We can then score different summaries, to indicate how "
+            "factual a summary is for a given article. The idea is that in production, you could generate a set of "
+            "summaries for the same article, with different parameters (or even different models). By using "
+            "post-processing error detection, we can then select the best possible summary.")
 if st.session_state.article_text:
     with st.spinner('Generating summary...'):
         # classify_comment(article_text, selected_model)
@@ -395,6 +302,8 @@ if is_valid_url(article_text):
     print("YES")
 else:
     print("NO")
+
+
 def render_svg(svg_file):
     with open(svg_file, "r") as f:
         lines = f.readlines()
@@ -408,11 +317,15 @@ def render_svg(svg_file):
 
 # ENTITY MATCHING PART
 st.header("Entity matching")
-st.markdown("**Named entity recognition** (NER) is the task of identifying and categorising key information ("
-            "entities) in text. An entity can be a singular word or a series of words that consistently refers to the "
-            "same thing. Common entity classes are person names, organisations, locations and so on. By applying NER "
-            "to both the article and its summary, we can spot possible **hallucinations**. Hallucinations are words "
-            "generated by the model that are not supported by the source input. ")
+st.markdown("The first method we will discuss is called **Named Entity Recognition** (NER). NER is the task of "
+            "identifying and categorising key information (entities) in text. An entity can be a singular word or a "
+            "series of words that consistently refers to the same thing. Common entity classes are person names, "
+            "organisations, locations and so on. By applying NER to both the article and its summary, we can spot "
+            "possible **hallucinations**. Hallucinations are words generated by the model that are not supported by "
+            "the source input. In theory all entities in the summary (such as dates, locations and so on), "
+            "should also be present in the article. Thus we can extract all entities from the summary and compare "
+            "them to the entities of the original article, spotting potential hallucinations. The more unmatched "
+            "entities we find, the lower the factualness score of the summary. ")
 with st.spinner("Calculating and matching entities..."):
     entity_match_html = highlight_entities(selected_article)
     st.write(entity_match_html, unsafe_allow_html=True)
@@ -424,31 +337,47 @@ with st.spinner("Calculating and matching entities..."):
 
     markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
     markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
-    st.markdown("Here you can see what this looks like when we apply entity-matching on the summary (compared to the "
-                "original article). Entities in this summary are marked  " + green_text + " when the entity also "
-                                                                                          "exists in the article, while unmatched entities are marked " + red_text + ".",
+    st.markdown("We call this technique “entity matching” and here you can see what this looks like when we apply "
+                "this method on the summary. Entities in the summary are marked  " + green_text + " when the entity "
+                                                                                                  "also exists in the "
+                                                                                                  "article, "
+                                                                                                  "while unmatched "
+                                                                                                  "entities are "
+                                                                                                  "marked " +
+                red_text + ". Several of the example articles and their summaries indicate different errors we find "
+                           "by using this technique. Based on which article you choose, we provide a short "
+                           "explanation of the results below.",
                 unsafe_allow_html=True)
     entity_specific_text = fetch_entity_specific_contents(selected_article)
-    st.markdown(entity_specific_text)
+    soup = BeautifulSoup(entity_specific_text, features="html.parser")
+    HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
+    margin-bottom: 2.5rem">{}</div> """
+    st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
 
 # DEPENDENCY PARSING PART
 st.header("Dependency comparison")
-st.markdown("**Dependency parsing** is the process in which the grammatical structure in a sentence is analysed, "
-            "to find out related words as well as the type of the relationship between them. For the sentence “Jan’s "
-            "wife is called Sarah” you would get the following dependency graph:")
+st.markdown("The second method we use for post-processing is called **Dependency parsing**: the process in which the "
+            "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "
+            "relationship between them. For the sentence “Jan’s wife is called Sarah” you would get the following "
+            "dependency graph:")
 
 # TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)
 # st.image("ExampleParsing.svg")
 st.write(render_svg('ExampleParsing.svg'), unsafe_allow_html=True)
 st.markdown("Here, “Jan” is the “poss” (possession modifier) of “wife”. If suddenly the summary would read “Jan’s "
-            "husband…”, there would be a dependency in the summary that is non-existent in the article itself. "
-            "However, it could be that such a new dependency is not per se correct, “The borders of Ukraine” have a "
-            "different dependency between “borders” and “Ukraine” than “Ukraine’s borders”, while this would also be "
-            "correct. So general matching between summary and article wont work.")
-st.markdown("There is however a simple method that we found has potential in post-processing. Based on empirical "
-            "results, we have found that when there are specific kinds of dependencies in the summary that are not in "
-            "the article, these specific types are often an indication of a wrongly constructed sentence. Let’s take "
-            "a look at an example:")
+            "husband…”, there would be a dependency in the summary that is non-existent in the article itself (namely "
+            "“Jan” is the “poss” of “husband”). However, often new dependencies are introduced in the summary that "
+            "are still correct. “The borders of Ukraine” have a different dependency between “borders” and “Ukraine” "
+            "than “Ukraine’s borders”, while both descriptions have the same meaning. So just matching all "
+            "dependencies between article and summary (as we did with entity matching) would not be a robust method.")
+st.markdown("However, by empirical testing, we have found that there are certain dependencies which can be used for "
+            "such matching techniques. When unmatched, these specific dependencies are often an indication of a "
+            "wrongly constructed sentence. **Should I explain this more/better or is it enough that I explain by "
+            "example specific run throughs?**. We found 2(/3 TODO) common dependencies which, when present in the "
+            "summary but not in the article, are highly indicative of factualness errors. Furthermore, we only check "
+            "dependencies between an existing **entity** and its direct connections. Below we highlight all unmatched "
+            "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
+            "currently selected article.")
 with st.spinner("Doing dependency parsing..."):
     summary_deps = check_dependency(False)
     article_deps = check_dependency(True)
@@ -461,22 +390,22 @@ with st.spinner("Doing dependency parsing..."):
     if total_unmatched_deps:
         for current_drawing_list in total_unmatched_deps:
             render_dependency_parsing(current_drawing_list)
-    dep_spec_text = fetch_dependency_specific_contents(selected_article)
-    st.markdown(dep_spec_text)
-    soup = BeautifulSoup("Example text option with box", features="html.parser")
+    dep_specific_text = fetch_dependency_specific_contents(selected_article)
+    soup = BeautifulSoup(dep_specific_text, features="html.parser")
     HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
     margin-bottom: 2.5rem">{}</div> """
-    st.write(HTML_WRAPPER.format(soup), unsafe_allow_html=True)
+    st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
 
 # OUTRO/CONCLUSION
 st.header("Wrapping up")
 st.markdown("We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
-            "be used to solve hallucinations, while checking if specific dependencies are matched between summary and "
-            "article can be used to filter out some bad sentences (and thus worse summaries). Of course these are "
-            "only basic methods which were empirically tested, but they are a start at actually making something good "
-            "(???). (something about that we tested also RE and maybe other things).")
+            "be used to solve hallucinations, while dependency comparison can be used to filter out some bad "
+            "sentences (and thus worse summaries). These methods highlight the possibilities of post-processing "
+            "AI-made summaries, but are only a basic introduction. As the methods were empirically tested they are "
+            "definitely not sufficiently robust for general use-cases. (something about that we tested also RE and "
+            "maybe other things).")
 st.markdown("####")
-st.markdown("Now based on these methods you can check summaries and whether they are “good” or “bad”. Below you can "
-            "generate 5 different kind of summaries for the starting article (based on different model params) in "
-            "which their ranks are estimated, and hopefully the best summary (read: the one that a human would prefer "
-            "or indicate as the best one) will be at the top.")
+st.markdown("Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
+            "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
+            "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
+            "the actual parameters or something? ")

dependency-specific-text/article11.txt

@@ -1,4 +1,4 @@
-One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "poss" (possession modifier) dependency.
-In the image above, you can see the unmatched dependency that is found in the summary but not present in the article. For the "poss" dependency, we only check matches when the target word is "in", as it is here. U.S. is the entity here.
-For this specific example, it's obvious that the dependency of "in U.S." is not found in the article, as you can already see in the entity matching paragraph that U.S. is a hallucinated entity and doesn't occur in the article itself,
-so technically we don't need dependency comparison here to spot this particular error.
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "pobj" (object of preposition) dependency.
+Furthermore, we only match *pobj* dependencies when the target word is "in", as in this example.
+In this case it's obvious that "in U.S." is not found in the article, as "U.S." is a hallucinated entity itself as discussed in the entity matching paragraph.
+So technically we don't need dependency comparison to spot the error from this summary.

dependency-specific-text/article13.txt

@@ -1,3 +1,9 @@
 One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "amod" (adjectival modifier) dependency.
-In the image above, you can see the unmatched dependency that is found in the summary but not present in the article. "First" is the entity here, and it's the adjectival modifier of the word "phone".
-However, this sentence is not factual, since the article talks about a **new** type of flagship phone, and not at all the **first** flagship phone. This is wrong, and the error was found by filtering on this specific kind of dependency.
+Applied to this summary, we have "First" as the entity, and it is the adjectival modifier of the word "phone".
+And indeed, this unmatched dependency indicates an actual error here. The sentence is not factual, since the article talks about a **new** type of flagship phone,
+and not the **first** flagship phone. This error was found by filtering on this specific kind of dependency. Empirical results showed that unmatched *amod* dependencies often suggest
+that the summary sentence contains an error.  <br> <br>
+Another dependency that we use is the "pobj" (object of preposition) dependency.
+Furthermore, we only match *pobj* dependencies when the target word is "in", as in this example.
+In this case the sentence itself contains a factual error (because the article states "there's no word on a US release date yet").
+However, this could have been found by entity matching already (as january 18 is unmatched), and the unmatched dependency can not be completely blamed for this error here.

dependency-specific-text/article4.txt

@@ -0,0 +1,7 @@
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "amod" (adjectival modifier) dependency.
+Applied to this summary, we have "Democratic" as the entity, and it is the adjectival modifier of the word "member".
+And indeed, this unmatched dependency indicates an actual error here. The sentence is not factual for two reasons. <br> <br>
+First, the article talks about "democrats" and "members of the committee", which are two separate things. The summary combines those two in a way
+that can be seen as not completely factual. Second, the statement itself was not made by a democrat (nor a member of the committee), and even though the dependency can't be
+directly linked to this error, empirical results showed that unmatched *amod* dependencies often suggest
+that the summary sentence is incorrect.

dependency-specific-text/article9.txt

@@ -0,0 +1,4 @@
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "pobj" (object of preposition) dependency.
+Furthermore, we only match *pobj* dependencies when the target word is "in", as in this example.
+The sentence here is not a factual error per se, but rather a readability issue. The "in" should be dropped to make the sentence correct.
+For better examples with this specific dependency, try choosing another article. TODO: readability issue with the dependency graph for this specific issue

entity-specific-text/article11.txt

@@ -1,3 +1,4 @@
-For this summary, there are 2 unmatched entities: "The Mark Levinson" and "U.S". The first one
-is not actually a real error per se, but rather a "the" before "Mark Levinson" (TODO EXPLAIN BIT BETTER).
-The "U.S." however is a hallucinated entity not present in the article, and via this method this can be found.
+As you can see we have 1 unmatched entity: "U.S." is a hallucinated entity in the summary, that does not exist in the article.
+Deep learning based generation is [prone to hallucinate](https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade
+system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem
+for the downstream task of summary generation.

entity-specific-text/article13.txt

@@ -1,2 +1,5 @@
-For this summary, there are 2 unmatched entities: "January 18" and "U.S". January 18 is indeed a hallucinated entity, as there is no sentence containing this exact date. U.S. does occur in the article, but as "US" instead of "U.S.". This can be solved
-by comparing to a list of abbreviations (of embeddings :TODO?)
+As you can see we have 2 unmatched entities: "January 18" and "U.S". The first one is a hallucinated entity in the summary, that does not exist in the article.
+Deep learning based generation is [prone to hallucinate](https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade
+system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem
+for the downstream task of summary generation. U.S. **does** occur in the article, but as "US" instead of "U.S.". This could be solved
+by comparing to a list of abbreviations or with a specific embedder for abbreviations but is currently not implemented.

entity-specific-text/article16.txt

@@ -0,0 +1,4 @@
+As you can see we have 1 unmatched entity: "Six9" is a hallucinated entity in the summary, that does not exist in the article.
+Deep learning based generation is [prone to hallucinate](https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade
+system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem
+for the downstream task of summary generation.

requirements.txt

@@ -2,6 +2,7 @@ beautifulsoup4==4.10.0
 streamlit==1.2.0
 transformers==4.15.0
 transformers-interpret==0.5.2
+sentence-transformers==2.2.0
 spacy==3.0.0
 spacy_streamlit==1.0.3
 flair

sample-articles/article4.txt

@@ -0,0 +1,55 @@
+Former White House chief of staff Mark Meadows will no longer cooperate with the House select committee investigating January 6 insurrection, according to a letter from his attorney to the panel, which was obtained by CNN on Tuesday.
+
+"We agreed to provide thousands of pages of responsive documents and Mr. Meadows was willing to appear voluntarily, not under compulsion of the Select Committee's subpoena to him, for a deposition to answer questions about non-privileged matters. Now actions by the Select Committee have made such an appearance untenable," the letter from George J. Terwilliger II stated.
+
+"In short, we now have every indication from the information supplied to us last Friday -- upon which Mr. Meadows could expect to be questioned -- that the Select Committee has no intention of respecting boundaries concerning Executive Privilege," Terwilliger added.
+
+The committee said later Tuesday that it will move forward with a scheduled deposition with Meadows on Wednesday even though he said he no longer plans to cooperate.
+
+By proceeding with the scheduled deposition, the committee is setting up a path to hold Meadows in criminal contempt.
+
+"Tomorrow's deposition, which was scheduled at Mr. Meadows's request, will go forward as planned. If indeed Mr. Meadows refuses to appear, the Select Committee will be left no choice but to advance contempt proceedings and recommend that the body in which Mr. Meadows once served refer him for criminal prosecution," Democratic Rep. Bennie Thompson of Mississippi and GOP Rep. Liz Cheney of Wyoming, who lead the committee, said in a joint statement.
+
+Thompson told CNN later Tuesday evening, "Obviously, we had hoped Mr. Meadows would continue to work with the committee. But obviously based on his lawyer's letter today and his plan to not show up for the deposition, that creates a different dynamic."
+
+"As you know, we were prepared to go with contempt earlier, but we withheld it based on what we thought was an agreement that we'd work together. That has not been the case. So obviously, we will move forward with it," he said.
+
+Although Thompson indicated criminal contempt was on the table, he made clear that the committee is weighing multiple options, including immunity, that could pave the way for it to get the information that it wants from Meadows.
+
+"I think we're interested in getting the information. I think we will still want Mr. Meadows to cooperate. So we will look at all of our options at this point," he said.
+
+Responding to the letter from Meadows' attorney, the committee made clear it needs to hear from the former White House chief of staff "about voluminous official records stored in his personal phone and email accounts, which were required to be turned over to the National Archives in accordance with the Presidential Records Act. "
+
+A source familiar with the matter told CNN that among the 6,000 pages of documents Meadows has already provided to the committee are communications from January 6. It is still unclear who communicated that day with Meadows but the source said that "many people had Meadows' cell phone."
+
+Democratic Rep. Pete Aguilar of California, who serves on the panel, told CNN that within the documents Meadows turned over is evidence that he was in communication with individuals involved in the planning of the rally on January 6 that preceded the riot.
+
+"What I'll share is that we continue to learn and we continue to connect the dots," Aguilar said. "But individuals that were responsible for the planning of January 6 in the rally, Mr. Meadows was in communication with, and those are in the documents ... that he turned over himself."
+
+Aguilar added that some of the records Meadows turned over, including text messages, were from his personal device.
+
+Rep. Zoe Lofgren, a California Democrat and member of the committee, said on CNN's "The Lead with Jake Tapper" that the records including "volumes of material, including real time communication as the riot unfolded. Lofgren said the messages were shared "without an assertion of privilege," and criticized Meadows for then reversing his cooperation.
+
+"The committee wants to ask him about some of that, and it's really untenable that all of a sudden at the last minute he's saying no. That somehow there's some reason why he can't talk about this," Lofgren said.
+
+CNN first reported last week that Meadows had begun cooperating with the committee, handing over thousands of documents and agreeing to appear for an interview this week.
+
+Meadows' about-face is due in part to learning over the weekend that the committee had "issued wide ranging subpoenas for information from a third party communications provider," the letter notes.
+
+"As a result of careful and deliberate consideration of these factors, we now must decline the opportunity to appear voluntarily for a deposition," Terwilliger writes.
+
+Terwilliger writes that Meadows would answer written questions "so that there might be both an orderly process and a clear record of questions and related assertions of privilege where appropriate."
+
+Responding to Meadows' claim that the committee was ignoring his claims of executive privilege, Thompson and Cheney state that Meadows was willing to discuss details about Trump in his new book.
+
+"Mark Meadows has informed the Select Committee that he does not intend to cooperate further with our investigation despite his apparent willingness to provide details about the facts and circumstances surrounding the January 6th attack, including conversations with President Trump, in the book he is now promoting and selling," they write.
+
+The pair add that they have "numerous questions" for Meadows that have nothing to do with executive privilege.
+
+Rep. Stephanie Murphy, a Florida Democrat who's also a member of the select committee, said Tuesday evening that while the panel will continue to do all it can to compel the testimony of witnesses like Meadows and Steve Bannon, members believe they'll be able to get the information they're looking for without their help.
+
+"To be fair, it's only a very handful of people who want to risk jail time and fines for contempt of Congress who are obstructing our process," Murphy said.
+
+"The vast majority of the people that we have reached out to are providing us with information, with evidence, with text messages, with emails, with details of conversations that they have been a party to. So these people are well within their right to not cooperate, but it's not as if we're not going to get to the information we need."
+
+This story has been updated with additional developments Tuesday.

sample-articles/article9.txt

@@ -0,0 +1,48 @@
+Novak Djokovic has been granted permission to defend his Australian Open title
+
+Australians have reacted angrily to news that tennis player Novak Djokovic will play in the Australian Open, after being exempted from vaccination rules.
+
+All players and staff at the tournament must be vaccinated or have an exemption granted by an expert independent panel.
+
+Djokovic has not spoken about his vaccination status, but last year said he was "opposed to vaccination".
+
+Organisers say he has not been given special treatment. But Australians have criticised officials and the player.
+
+The controversy comes as the country is seeing tens of thousands of Covid-19 cases for the first time after enduring some of the world's strictest restrictions.
+
+Over 90% of Australia's over-16 population is fully vaccinated, but some Australians still cannot travel interstate or globally because of current restrictions.
+
+Amid the row, Australian Prime Minister Scott Morrison said Djokovic would be required to present evidence upon arrival that he has a genuine medical exemption from vaccination, or he would be "on the next plane home".
+
+"If that evidence is insufficient, then he won't be treated any different to anyone else and he'll be on the next plane home," the prime minister told reporters. "There should be no special rules for Novak Djokovic at all. None whatsoever."
+
+Many Australians had previously accused the government of allowing the rich and famous to do as they please while ordinary people remained separated from sick and dying loved ones.
+
+"I think it's a disgrace," Christine Wharton, who lives in Melbourne, where the Australian Open will be held, told ABC.
+
+"We've all done the right thing, we've all gone out and got our jabs and our boosters and we have someone that has come from overseas and all of a sudden he's been exempt and can play and I think it's an absolute disgrace and I won't be watching it."
+
+A&E doctor Stephen Parnis tweeted: "I don't care how good a tennis player he is. If he's refusing to get vaccinated, he shouldn't be allowed in. "If this exemption is true, it sends an appalling message to millions seeking to reduce #COVID19Aus risk to themselves & others."
+
+The decision raised eyebrows with some other tennis players too. "I just think it's very interesting. That's all I'm going to say," Australian Alex de Minaur said.
+
+Britain's Jamie Murray added: "I think if it was me that wasn't vaccinated I wouldn't be getting an exemption. You know, but well done to him for getting clear to come to Australia and compete."
+
+The Australian Open begins on 17 January, and the event's chief executive Craig Tiley said 26 athletes had applied for medical exemptions. "A handful" had been granted, he said, under guidelines set by federal regulators.
+
+"We made it extra difficult for anyone applying for an application to ensure it was the right process and to make sure the medical experts deal with it independently," he told Channel 9.
+
+Applications for medical exemptions are being assessed anonymously by two separate panels, with inflammatory cardiac illness or another acute condition listed as valid reasons.
+
+But it is also possible Djokovic has recently tested positive for the virus, which would allow him to defer taking the vaccine.
+
+He has not revealed his vaccination status and said last April: "Personally I am opposed to vaccination and I wouldn't want to be forced by someone to take a vaccine in order to be able to travel."
+
+On Tuesday he said on Instagram: "I've spent fantastic quality time with my loved ones over the break and today I'm heading down under with an exemption permission. Let's go 2022. I am ready to live and breathe tennis in the next few weeks of competition."
+
+Media caption,
+"Heartless" Queensland bars US couple from seeing dying father
+
+Victoria state government minister Jaala Pulford acknowledged the decision was "frustrating and upsetting", but also denied that Djokovic had received special treatment. Both she and Mr Tiley urged Djokovic to give more information to the public.
+
+"It'll certainly be helpful if Novak was to explain the conditions in which he's sought an exemption and granted an exemption but ultimately it's up to him," Mr Tiley said.

sample-summaries/article4.txt

@@ -0,0 +1 @@
+Former White House chief of staff Mark Meadows will no longer cooperate with the House select committee. The committee says it will move forward with a scheduled deposition with Meadows on Wednesday. By proceeding with the scheduled deposition, the committee is setting up a path to hold Meadows in criminal contempt. A source familiar with the matter told CNN that among the 6,000 pages of documents Meadows has already provided to the committee are communications from January 6. A Democratic member of the committee said Meadows' about-face is due in part to learning over the weekend that the committee had "issued wide ranging subpoenas for information from a third party communications provider".

sample-summaries/article9.txt

@@ -0,0 +1 @@
+Novak Djokovic has been granted permission to play in the Australian Open. All players and staff at the tournament must be vaccinated or have an exemption.. Djokovic has not spoken about his vaccination status, but last year said he was "opposed to vaccination" Australian Prime Minister Scott Morrison says there should be no special rules for Djokovic, but adds he would be "on the next plane home" if he did not have the right evidence. in Australia is seeing tens of thousands of Covid-19 cases for the first time after enduring some of the world's strictest restrictions.