Added multi-sentence support

.gitignore

@@ -157,4 +157,6 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/
+
+*.conllu

app.py

@@ -1,5 +1,7 @@
 import logging
 import os
+import tempfile
+from pathlib import Path
 from typing import List, Tuple
 
 import gradio as gr
@@ -8,6 +10,8 @@ import spacy
 import torch
 from transformers import AutoModelForTokenClassification, AutoTokenizer
 
+from preprocessing import expand_contractions
+
 try:
     nlp = spacy.load("pt_core_news_sm")
 except Exception:
@@ -58,6 +62,7 @@ def predict(text, nlp, logger=None) -> Tuple[List[str], List[str]]:
 
 
 def text_analysis(text):
+    text = expand_contractions(text)
     tokens, labels, scores = predict(text, nlp, logger)
     pos_count = pd.DataFrame(
         {
@@ -79,30 +84,85 @@ def text_analysis(text):
     }
 
 
+def batch_analysis(input_file):
+    text = open(input_file.name, encoding="utf-8").read()
+    text = text.split("\n")
+    name = Path(input_file.name).stem
+    sents = []
+    for sent in text:
+        sub_sents = nlp(sent).sents
+        sub_sents = [str(_sent).strip() for _sent in sub_sents]
+        sents += sub_sents
+    conllu_output = []
+
+    for i, sent in enumerate(sents):
+        conllu_output.append("# sent_id = {}-{}\n".format(name, i + 1))
+        conllu_output.append("# text = {}\n".format(sent))
+        tokens, labels, scores = predict(sent, nlp, logger)
+        for j, (token, label) in enumerate(zip(tokens, labels)):
+            conllu_output.append(
+                "{}\t{}\t_\t{}".format(j + 1, token, label) + "\t_" * 5 + "\n"
+            )
+        conllu_output.append("\n")
+
+    output_filename = "output.conllu"
+    with open(output_filename, "w") as out_f:
+        out_f.writelines(conllu_output)
+
+    return {output_file: output_file.update(visible=True, value=output_filename)}
+
+
 css = open("style.css").read()
 top_html = open("top.html").read()
 bottom_html = open("bottom.html").read()
 
 with gr.Blocks(css=css) as demo:
     gr.HTML(top_html)
-    text = gr.Textbox(placeholder="Enter your text here...", label="Input")
-    examples = gr.Examples(
-        examples=[
-            [
-                "A população não poderia ter acesso a relatórios que explicassem, por exemplo, os motivos exatos de atrasos em obras de linhas e estações."
+    with gr.Tab("Single sentence"):
+        text = gr.Textbox(placeholder="Enter your text here...", label="Input")
+        examples = gr.Examples(
+            examples=[
+                [
+                    "A população não poderia ter acesso a relatórios que explicassem, por exemplo, os motivos exatos de atrasos em obras de linhas e estações."
+                ],
+                [
+                    "Filme 'Star Wars : Os Últimos Jedi' ganha trailer definitivo; assista."
+                ],
             ],
-            ["Filme 'Star Wars : Os Últimos Jedi' ganha trailer definitivo; assista."],
-        ],
-        inputs=[text],
-        label="Select an example",
-    )
-    output_highlighted = gr.HighlightedText(label="Colorful output", visible=False)
-    output_df = gr.Dataframe(label="Tabular output", visible=False)
-    submit_btn = gr.Button("Send")
-    submit_btn.click(
-        fn=text_analysis, inputs=text, outputs=[output_highlighted, output_df]
-    )
+            inputs=[text],
+            label="Select an example",
+        )
+        output_highlighted = gr.HighlightedText(label="Colorful output", visible=False)
+        output_df = gr.Dataframe(label="Tabular output", visible=False)
+        submit_btn = gr.Button("Send")
+        submit_btn.click(
+            fn=text_analysis, inputs=text, outputs=[output_highlighted, output_df]
+        )
+    with gr.Tab("Multiple sentences"):
+        gr.HTML(
+            """
+        <p>Upload file with raw sentences in it. Below is an example of what we expect the contents of the file to look like.
+        Sentences are automatically splitted by Spacy's sentencizer.
+        To force an explicit division, manually separate the sentences on different lines.</p>
+        """
+        )
+        gr.Markdown(
+            """
+        ```
+        Então ele hesitou, quase como se estivesse surpreso com as próprias palavras, e recitou:
+        – Vá e não tornes a pecar!
+        Baley, sorrindo de repente, pegou no cotovelo de R. Daneel e eles saíram juntos pela porta.
+        ```
+        """
+        )
+        input_file = gr.File(label="Upload your input file here...")
+        output_file = gr.File(visible=False)
+        submit_btn_batch = gr.Button("Send")
+        submit_btn_batch.click(
+            fn=batch_analysis, inputs=input_file, outputs=output_file
+        )
+
     gr.HTML(bottom_html)
 
 
-demo.launch(debug=True)
+demo.launch(debug=True, server_port=15000)

preprocessing.py

@@ -0,0 +1,109 @@
+import re
+
+contractions = {
+    r"(?<![\w.])no(s)?(?![$\w])": r"em o\g<1>",
+    r"(?<![\w.])na(s)?(?![$\w])": r"em a\g<1>",
+    r"(?<![\w.])da(s)?(?![$\w])": r"de a\g<1>",
+    r"(?<![\w.])do(s)?(?![$\w])": r"de o\g<1>",
+    r"(?<![\w.])ao(s)?(?![$\w])": r"a o\g<1>",
+    r"(?<![\w.])à(s)?(?![$\w])": r"a a\g<1>",
+    r"(?<![\w.])pela(s)?(?![$\w])": r"por a\g<1>",
+    r"(?<![\w.])pelo(s)?(?![$\w])": r"por o\g<1>",
+    r"(?<![\w.])nesta(s)?(?![$\w])": r"em esta\g<1>",
+    r"(?<![\w.])neste(s)?(?![$\w])": r"em este\g<1>",
+    r"(?<![\w.])nessa(s)?(?![$\w])": r"em essa\g<1>",
+    r"(?<![\w.])nesse(s)?(?![$\w])": r"em esse\g<1>",
+    r"(?<![\w.])num(?![$\w])": r"em um",
+    r"(?<![\w.])nuns(?![$\w])": r"em uns",
+    r"(?<![\w.])numa(s)?(?![$\w])": r"em uma\g<1>",
+    r"(?<![\w.])nisso(?![$\w])": r"em isso",
+    r"(?<![\w.])naquele(s)?(?![$\w])": r"em aquele\g<1>",
+    r"(?<![\w.])naquela(s)?(?![$\w])": r"em aquela\g<1>",
+    r"(?<![\w.])naquilo(?![$\w])": r"em aquelo",
+    r"(?<![\w.])duma(s)?(?![$\w])": r"de uma\g<1>",
+    r"(?<![\w.])daqui(?![$\w])": r"de aqui",
+    r"(?<![\w.])dali(?![$\w])": r"de ali",
+    r"(?<![\w.])daquele(s)?(?![$\w])": r"de aquele\g<1>",
+    r"(?<![\w.])daquela(s)?(?![$\w])": r"de aquela\g<1>",
+    r"(?<![\w.])deste(s)?(?![$\w])": r"de este\g<1>",
+    r"(?<![\w.])desta(s)?(?![$\w])": r"de esta\g<1>",
+    r"(?<![\w.])desse(s)?(?![$\w])": r"de esse\g<1>",
+    r"(?<![\w.])dessa(s)?(?![$\w])": r"de essa\g<1>",
+    r"(?<![\w.])daí(?![$\w])": r"de aí",
+    r"(?<![\w.])dum(?![$\w])": r"de um",
+    r"(?<![\w.])donde(?![$\w])": r"de onde",
+    r"(?<![\w.])disto(?![$\w])": r"de isto",
+    r"(?<![\w.])disso(?![$\w])": r"de isso",
+    r"(?<![\w.])daquilo(?![$\w])": r"de aquilo",
+    r"(?<![\w.])dela(s)?(?![$\w])": r"de ela\g<1>",
+    r"(?<![\w.])dele(s)?(?![$\w])": r"de ele\g<1>",
+    r"(?<![\w.])nisto(?![$\w])": r"em isto",
+    r"(?<![\w.])nele(s)?(?![$\w])": r"em ele\g<1>",
+    r"(?<![\w.])nela(s)?(?![$\w])": r"em ela\g<1>",
+    r"(?<![\w.])d'?ele(s)?(?![$\w])": r"de ele\g<1>",
+    r"(?<![\w.])d'?ela(s)?(?![$\w])": r"de ela\g<1>",
+    r"(?<![\w.])noutro(s)?(?![$\w])": r"em outro\g<1>",
+    r"(?<![\w.])aonde(?![$\w])": r"a onde",
+    r"(?<![\w.])àquela(s)?(?![$\w])": r"a aquela\g<1>",
+    r"(?<![\w.])àquele(s)?(?![$\w])": r"a aquele\g<1>",
+    r"(?<![\w.])àquilo(?![$\w])": r"a aquelo",
+    r"(?<![\w.])contigo(?![$\w])": r"com ti",
+    r"(?<![\w.])né(?![$\w])": r"não é",
+    r"(?<![\w.])comigo(?![$\w])": r"com mim",
+    r"(?<![\w.])contigo(?![$\w])": r"com ti",
+    r"(?<![\w.])conosco(?![$\w])": r"com nós",
+    r"(?<![\w.])consigo(?![$\w])": r"com si",
+    r"(?<![\w.])pra(?![$\w])": r"para a",
+    r"(?<![\w.])pro(?![$\w])": r"para o",
+}
+
+
+def replace_keep_case(word, replacement, text):
+    """
+    Custom function for replace keeping the original case.
+    Parameters
+    ----------
+    word: str
+        Text to be replaced.
+    replacement: str
+        String to replace word.
+    text:
+        Text to be processed.
+    Returns
+    -------
+    str:
+        Processed string
+    """
+
+    def func(match):
+        g = match.group()
+        repl = match.expand(replacement)
+        if g.islower():
+            return repl.lower()
+        if g.istitle():
+            return repl.capitalize()
+        if g.isupper():
+            return repl.upper()
+        return repl
+
+    return re.sub(word, func, text, flags=re.I)
+
+
+def expand_contractions(text: str) -> str:
+    """
+    Replace contractions to their based form.
+    Parameters
+    ----------
+    text: str
+        Text that may contain contractions.
+    Returns
+    -------
+    str:
+        Text with expanded contractions.
+    """
+
+    for contraction in contractions.keys():
+        replace_str = contractions[contraction]
+        text = replace_keep_case(contraction, replace_str, text)
+
+    return text

style.css

@@ -14,7 +14,7 @@ a {
 }
 
 .container {
-    max-width: 730px;
+    max-width: 900px;
     margin: auto;
     padding-top: 1.5rem;
 }