Fixed contraction format and added jupyter tagger version

.dockerignore

@@ -0,0 +1,24 @@
+**/.classpath
+**/.dockerignore
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

.env.example

@@ -0,0 +1,7 @@
+DEFAULT_MODEL=Nome do modelo aqui (News / Tweets (stock market) / Oil and Gas (academic texts) / Multigenre)
+ID_COLUMN=nome da coluna com os ids dos tweets
+CONTENT_COLUMN=nome da coluna com o conteudo dos tweets
+PREFIX=prefixo para adicionar ao id dos tweets
+DATA_PATH=caminho para o arquivos .csv dos tweets
+OUTPUT_PATH=caminho para o arquivo de saída .conllu
+KEEP_REPLACE_CONTRACTION=Se a forma original da contrações deve ser mantida (True/False)

.gitattributes

@@ -31,4 +31,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -159,4 +159,8 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-*.conllu
+*.conllu
+
+/data
+/output
+.env

Dockerfile

@@ -0,0 +1,21 @@
+# Use a imagem oficial do Python 3.10 como base
+FROM python:3.10-slim
+
+# Defina o diretório de trabalho como /app
+WORKDIR /app
+
+# Copie o arquivo de requisitos para o diretório de trabalho
+COPY requirements.txt .
+
+# Instale as dependências do Python
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copie o seu arquivo Python para o diretório de trabalho
+COPY . .
+
+# Comando padrão para executar seu arquivo Python quando o contêiner for iniciado
+CMD ["python", "main.py"]
+
+#Execute os seguintes comandos em ordem:
+#docker build -t porttaggerdante .
+#docker run -v "caminho pro arquivo de saída:/app/output" porttaggerdante

README.md

@@ -1,7 +1,7 @@
 ---
-title: Porttagger
+title: Porttagger-DANTE
 emoji: ✍️
-colorFrom: purple
+colorFrom: orange
 colorTo: purple
 sdk: gradio
 sdk_version: 3.9.1

app.py

@@ -7,10 +7,12 @@ import gradio as gr
 import pandas as pd
 import spacy
 import torch
+
 from dante_tokenizer import DanteTokenizer
+
 from transformers import AutoModelForTokenClassification, AutoTokenizer
 
-from preprocessing import expand_contractions
+from preprocessing import *
 
 try:
     nlp = spacy.load("pt_core_news_sm")
@@ -19,7 +21,7 @@ except Exception:
     nlp = spacy.load("pt_core_news_sm")
 dt_tokenizer = DanteTokenizer()
 
-default_model = "News"
+default_model = "Tweets (stock market)"
 model_choices = {
     "News": "Emanuel/porttagger-news-base",
     "Tweets (stock market)": "Emanuel/porttagger-tweets-base",
@@ -90,59 +92,37 @@ def predict(text, logger=None) -> Tuple[List[str], List[str]]:
     return tokens, labels, scores
 
 
-def text_analysis(text):
-    text = expand_contractions(text)
-    tokens, labels, scores = predict(text, logger)
-    if len(labels) != len(tokens):
-        m = len(tokens) - len(labels)
-        labels += [None] * m
-        scores += [0] * m
-    pos_count = pd.DataFrame(
-        {
-            "token": tokens,
-            "tag": labels,
-            "confidence": scores,
-        }
-    )
-    pos_tokens = []
-    for token, label in zip(tokens, labels):
-        pos_tokens.extend([(token, label), (" ", None)])
-
-    output_highlighted.update(visible=True)
-    output_df.update(visible=True)
-
-    return {
-        output_highlighted: output_highlighted.update(visible=True, value=(pos_tokens)),
-        output_df: output_df.update(visible=True, value=pos_count),
-    }
-
-
-def batch_analysis(input_file):
-    text = open(input_file.name, encoding="utf-8").read()
-    text = text.split("\n")
-    name = Path(input_file.name).stem
-    sents = []
-    for sent in text:
-        sub_sents = nlp(sent).sents
-        sub_sents = [str(_sent).strip() for _sent in sub_sents]
-        sents += sub_sents
+def batch_analysis_csv(input_file, id_column: str='tweet_id', content_column: str='content', prefix: str='dante_02', keep_replace_contraction=True):
+    df = pd.read_csv(input_file.name, encoding='utf-8')
+    ids = df[id_column]
+    texts = df[content_column]
+    texts = texts.replace(r'\\n', ' ', regex=True)
+    texts = texts.apply(lambda x : x.strip())
     conllu_output = []
 
-    for i, sent in enumerate(sents):
-        sent = expand_contractions(sent)
-        conllu_output.append("# sent_id = {}-{}\n".format(name, i + 1))
+    for id, sent in zip(ids, texts):
+        conllu_output.append("# sent_id = {}_{}\n".format(prefix, id))
         conllu_output.append("# text = {}\n".format(sent))
-        tokens, labels, scores = predict(sent, logger)
-        for j, (token, label) in enumerate(zip(tokens, labels)):
-            conllu_output.append(
-                "{}\t{}\t_\t{}".format(j + 1, token, label) + "\t_" * 5 + "\n"
-            )
+        tokens, labels, _ = predict(sent, logger)
+        tokens_labels = list(zip(tokens, labels))
+
+        for j, (token, label) in enumerate(tokens_labels):
+            try:
+                contr = tokens_labels[j][0] + ' ' + tokens_labels[j+1][0]
+                for expansion in expansions.keys():
+                    replace_str = expansions[expansion]
+                    match = re.match(expansion, contr, re.I)
+                    expansion = replace_keep_case(expansion, replace_str, contr)
+                    if match is not None:
+                        conllu_output.append("{}\t{}".format(str(j+1)+'-'+str(j+2), expansion) + "\t_" * 8 + "\n")
+                        break
+                conllu_output.append("{}\t{}\t_\t{}".format(j + 1, token, label) + "\t_" * 6 + "\n")
+            except IndexError:
+                conllu_output.append("{}\t{}\t_\t{}".format(j + 1, token, label) + "\t_" * 6 + "\n") 
         conllu_output.append("\n")
-
     output_filename = "output.conllu"
-    with open(output_filename, "w") as out_f:
+    with open(output_filename, "w", encoding='utf-8') as out_f:
         out_f.writelines(conllu_output)
-
     return {output_file: output_file.update(visible=True, value=output_filename)}
 
 
@@ -154,26 +134,11 @@ with gr.Blocks(css=css) as demo:
     gr.HTML(top_html)
     select_model = gr.Dropdown(choices=list(model_choices.keys()), label="Tagger model", value=default_model)
     select_model.change(myapp.load_model, inputs=[select_model])
-    with gr.Tab("Single sentence"):
-        text = gr.Textbox(placeholder="Enter your text here...", label="Input")
-        examples = gr.Examples(
-            examples=[
-                [
-                    "A população não poderia ter acesso a relatórios que explicassem, por exemplo, os motivos exatos de atrasos em obras de linhas e estações."
-                ],
-                [
-                    "Filme 'Star Wars : Os Últimos Jedi' ganha trailer definitivo; assista."
-                ],
-            ],
-            inputs=[text],
-            label="Select an example",
-        )
-        output_highlighted = gr.HighlightedText(label="Colorful output", visible=False)
-        output_df = gr.Dataframe(label="Tabular output", visible=False)
-        submit_btn = gr.Button("Tag it")
-        submit_btn.click(
-            fn=text_analysis, inputs=text, outputs=[output_highlighted, output_df]
-        )
+    
+    id_column = gr.Textbox(placeholder='tweet_id', label='Id column')
+    content_column = gr.Textbox(placeholder='content', label='Content column')
+    label_prefix = gr.Textbox(placeholder='dante_02', label='Label prefix')
+
     with gr.Tab("Multiple sentences"):
         gr.HTML(
             """
@@ -197,10 +162,10 @@ with gr.Blocks(css=css) as demo:
         output_file = gr.File(label="Tagged file", visible=False)
         submit_btn_batch = gr.Button("Tag it")
         submit_btn_batch.click(
-            fn=batch_analysis, inputs=input_file, outputs=output_file
+            fn=batch_analysis_csv, inputs=[input_file, id_column], outputs=output_file
         )
 
     gr.HTML(bottom_html)
 
 
-demo.launch(debug=True)
+demo.launch(debug=True)

main.py

@@ -0,0 +1,132 @@
+import logging
+import os
+from typing import List, Tuple
+import pandas as pd
+import spacy
+import torch
+from dante_tokenizer import DanteTokenizer
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+from dotenv import dotenv_values
+
+from dante_tokenizer.data.preprocessing import split_monetary_tokens, normalize_text, split_enclisis
+from preprocessing import *
+
+try:
+    nlp = spacy.load("pt_core_news_sm")
+except Exception:
+    os.system("python -m spacy download pt_core_news_sm")
+    nlp = spacy.load("pt_core_news_sm")
+dt_tokenizer = DanteTokenizer()
+
+model_choices = {
+    "News": "Emanuel/porttagger-news-base",
+    "Tweets (stock market)": "Emanuel/porttagger-tweets-base",
+    "Oil and Gas (academic texts)": "Emanuel/porttagger-oilgas-base",
+    "Multigenre": "Emanuel/porttagger-base",
+}
+pre_tokenizers = {
+    "News": nlp,
+    "Tweets (stock market)": dt_tokenizer.tokenize,
+    "Oil and Gas (academic texts)": nlp,
+    "Multigenre": nlp,
+}
+
+env_vars = dotenv_values('.env')
+
+for key, value in env_vars.items():
+    globals()[key] = value
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+class MyApp:
+    def __init__(self) -> None:
+        self.model = None
+        self.tokenizer = None
+        self.pre_tokenizer = None
+        self.load_model()
+
+    def load_model(self, model_name: str = DEFAULT_MODEL):
+        if model_name not in model_choices.keys():
+            logger.error("Selected model is not supported, resetting to the default model.")
+            model_name = DEFAULT_MODEL
+        self.model = AutoModelForTokenClassification.from_pretrained(model_choices[model_name])
+        self.tokenizer = AutoTokenizer.from_pretrained(model_choices[model_name])
+        self.pre_tokenizer = pre_tokenizers[model_name]
+
+myapp = MyApp()
+
+def predict(text, logger=None) -> Tuple[List[str], List[str]]:
+    doc = myapp.pre_tokenizer(text)
+    tokens = [token.text if not isinstance(token, str) else token for token in doc]
+
+    logger.info("Starting predictions for sentence: {}".format(text))
+    print("Using model {}".format(myapp.model.config.__dict__["_name_or_path"]))
+
+    input_tokens = myapp.tokenizer(
+        tokens,
+        return_tensors="pt",
+        is_split_into_words=True,
+        return_offsets_mapping=True,
+        return_special_tokens_mask=True,
+        
+    )
+    output = myapp.model(input_tokens["input_ids"])
+
+    i_token = 0
+    labels = []
+    scores = []
+    for off, is_special_token, pred in zip(
+        input_tokens["offset_mapping"][0],
+        input_tokens["special_tokens_mask"][0],
+        output.logits[0],
+    ):
+        if is_special_token or off[0] > 0:
+            continue
+        label = myapp.model.config.__dict__["id2label"][int(pred.argmax(axis=-1))]
+        if logger is not None:
+            logger.info("{}, {}, {}".format(off, tokens[i_token], label))
+        labels.append(label)
+        scores.append(
+            "{:.2f}".format(100 * float(torch.softmax(pred, dim=-1).detach().max()))
+        )
+        i_token += 1
+
+    return tokens, labels, scores
+
+def batch_analysis_csv(ID_COLUMN: str, CONTENT_COLUMN: str, DATA_PATH: str, PREFIX:str, OUTPUT_PATH: str, KEEP_REPLACE_CONTRACTION: bool):
+    df = pd.read_csv(DATA_PATH)
+    ids = df[ID_COLUMN]
+    texts = df[CONTENT_COLUMN]
+    texts = texts.replace(r'\\n', ' ', regex=True) # remover '\n' mas não por espaço
+    texts = texts.apply(lambda x : x.strip()) # remover espaços excedentes
+    conllu_output = []
+
+    for id, sent in zip(ids, texts):
+        conllu_output.append("# sent_id = {}_{}\n".format(PREFIX, id))
+        conllu_output.append("# text = {}\n".format(sent))
+        tokens, labels, _ = predict(sent, logger)
+        tokens_labels = list(zip(tokens, labels))
+
+        for j, (token, label) in enumerate(tokens_labels):
+            try:
+                contr = tokens_labels[j][0] + ' ' + tokens_labels[j+1][0]
+                for expansion in expansions.keys():
+                    replace_str = expansions[expansion]
+                    match = re.match(expansion, contr, re.IGNORECASE)
+                    expansion = replace_keep_case(expansion, replace_str, contr)
+                    if match is not None:
+                        conllu_output.append("{}\t{}".format(str(j+1)+'-'+str(j+2), expansion) + "\t_" * 8 + "\n")
+                        break
+                conllu_output.append("{}\t{}\t_\t{}".format(j + 1, token, label) + "\t_" * 6 + "\n")
+            except IndexError:
+                conllu_output.append("{}\t{}\t_\t{}".format(j + 1, token, label) + "\t_" * 6 + "\n")
+        conllu_output.append("\n")
+    with open(OUTPUT_PATH, 'w', encoding='utf-8') as out_f:
+        out_f.writelines(conllu_output)
+
+def main():
+    batch_analysis_csv(ID_COLUMN, CONTENT_COLUMN, DATA_PATH, PREFIX, OUTPUT_PATH, KEEP_REPLACE_CONTRACTION)
+
+if __name__ == '__main__':
+    main()

preprocessing.py

@@ -46,7 +46,7 @@ contractions = {
     r"(?<![\w.])aonde(?![$\w])": r"a onde",
     r"(?<![\w.])àquela(s)?(?![$\w])": r"a aquela\g<1>",
     r"(?<![\w.])àquele(s)?(?![$\w])": r"a aquele\g<1>",
-    r"(?<![\w.])àquilo(?![$\w])": r"a aquelo",
+    r"(?<![\w.])àquilo(?![$\w])": r"a aquilo",
     r"(?<![\w.])contigo(?![$\w])": r"com ti",
     r"(?<![\w.])né(?![$\w])": r"não é",
     r"(?<![\w.])comigo(?![$\w])": r"com mim",
@@ -58,6 +58,60 @@ contractions = {
 }
 
 
+expansions = {
+    r'^em o(s)?$': r'no\g<1>',
+    r'^em a(s)?$': r'na\g<1>',
+    r'^de a(s)?$': r'da\g<1>',
+    r'^de o(s)?$': r'do\g<1>',
+    r'^a o(s)?$': r'ao\g<1>',
+    r'^a a(s)?$': r'à\g<1>',
+    r'^por a(s)?$': r'pela\g<1>',
+    r'^por o(s)?$': r'pelo\g<1>',
+    r'^em esta(s)?$': r'nesta\g<1>',
+    r'^em este(s)?$': r'neste\g<1>',
+    r'^em essa(s)?$': r'nessa\g<1>',
+    r'^em esse(s)?$': r'nesse\g<1>',
+    r'^em um$': r'num',
+    r'^em uns$': r'nuns',
+    r'^em uma(s)?$': r'numa\g<1>',
+    r'^em isso$': r'nisso',
+    r'^em aquele(s)?$': r'naquele\g<1>',
+    r'^em aquela(s)?$': r'naquela\g<1>',
+    r'^em aquilo$': r'naquilo',
+    r'^de uma(s)?$': r'duma\g<1>',
+    r'^de aqui$': r'daqui',
+    r'^de ali$': r'dali',
+    r'^de aquele(s)?$': r'daquele\g<1>',
+    r'^de aquela(s)?$': r'daquela\g<1>',
+    r'^de este(s)?$': r'deste\g<1>',
+    r'^de esta(s)?$': r'desta\g<1>',
+    r'^de esse(s)?$': r'desse\g<1>',
+    r'^de essa(s)?$': r'dessa\g<1>',
+    r'^de aí$': r'daí',
+    r'^de um$': r'dum',
+    r'^de onde$': r'donde',
+    r'^de isto$': r'disto',
+    r'^de isso$': r'disso',
+    r'^de aquilo$': r'daquilo',
+    r'^de ela(s)?$': r"dela\g<1>",
+    r'^de ele(s)?$': r"dele\g<1>",
+    r'^em isto$': r'nisto',
+    r'^em ele(s)?$': r'nele\g<1>',
+    r'^em ela(s)?$': r'nela\g<1>',
+    r'^em outro(s)?$': r'noutro\g<1>',
+    r'^a onde$': r'aonde',
+    r'^a aquela(s)?$': r'àquela\g<1>',
+    r'^a aquele(s)?$': r'àquele\g<1>',
+    r'^a aquilo$': r'àquilo',
+    r'^com ti$': r'contigo',
+    r'^não é$': r'né',
+    r'^com mim$': r'comigo',
+    r'^com nós$': r'conosco',
+    r'^com si$': r'consigo',
+    r'^para a$': r'pra',
+    r'^para o$': r'pro'
+}
+
 def replace_keep_case(word, replacement, text):
     """
     Custom function for replace keeping the original case.
@@ -84,6 +138,8 @@ def replace_keep_case(word, replacement, text):
             return repl.capitalize()
         if g.isupper():
             return repl.upper()
+        if g[0].isupper():
+            return repl[0].upper() + repl[1:]
         return repl
 
     return re.sub(word, func, text, flags=re.I)

requirements.txt

@@ -57,6 +57,7 @@ pydub==0.25.1
 PyNaCl==1.5.0
 pyparsing==3.0.9
 python-dateutil==2.8.2
+python-dotenv==1.0.1
 python-multipart==0.0.5
 pytz==2022.6
 PyYAML==6.0
@@ -84,3 +85,4 @@ uvicorn==0.19.0
 wasabi==0.10.1
 websockets==10.4
 yarl==1.8.1
+option==2.1.0

top.html

@@ -1,7 +1,7 @@
 <div style="text-align: center; max-width: 650px; margin: 0 auto;">
     <div>
         <h1 style="font-weight: 900; font-size: 3rem; margin: 20px;">
-            Porttagger
+            PorttaggerDANTE
         </h1>
         <p class="slogan">A Brazilian Portuguese part of speech tagger according to the <a
                 href="https://universaldependencies.org/">Universal Dependencies</a> model