[#9] idiomifier:m-1-3 is ready. main_deploy.py is updated accordingly

explore/explore_bart_tokenizer_decode_idiom_special_tokens.py

@@ -0,0 +1,14 @@
+from idiomify.fetchers import fetch_tokenizer
+
+
+def main():
+    tokenizer = fetch_tokenizer("t-1-1")
+    sent = "There will always be a <idiom> silver lining </idiom> even when things look pitch black"
+    ids = tokenizer(sent)['input_ids']
+    print(ids)
+    decoded = tokenizer.decode(ids)
+    print(decoded)
+
+
+if __name__ == '__main__':
+    main()

idiomify/pipeline.py

@@ -1,3 +1,4 @@
+import re
 from typing import List
 from transformers import BartTokenizer
 from idiomify.builders import SourcesBuilder
@@ -18,5 +19,9 @@ class Pipeline:
             decoder_start_token_id=self.model.hparams['bos_token_id'],
             max_length=max_length,
         )  # -> (N, L_t)
-        tgts = self.builder.tokenizer.batch_decode(pred_ids, skip_special_tokens=True)
+        tgts = self.builder.tokenizer.batch_decode(pred_ids, skip_special_tokens=False)
+        tgts = [
+            re.sub(r"<s>|</s>", "", tgt)
+            for tgt in tgts
+        ]
         return tgts

main_deploy.py

@@ -1,9 +1,9 @@
 """
 we deploy the pipeline via streamlit.
 """
+import re
 import streamlit as st
-from transformers import BartTokenizer
-from idiomify.fetchers import fetch_config, fetch_idiomifier, fetch_idioms
+from idiomify.fetchers import fetch_config, fetch_idiomifier, fetch_idioms, fetch_tokenizer
 from idiomify.pipeline import Pipeline
 
 
@@ -11,7 +11,7 @@ from idiomify.pipeline import Pipeline
 def fetch_resources() -> tuple:
     config = fetch_config()['idiomifier']
     model = fetch_idiomifier(config['ver'])
-    tokenizer = BartTokenizer.from_pretrained(config['bart'])
+    tokenizer = fetch_tokenizer(config['tokenizer_ver'])
     idioms = fetch_idioms(config['idioms_ver'])
     return config, model, tokenizer, idioms
 
@@ -23,17 +23,20 @@ def main():
     pipeline = Pipeline(model, tokenizer)
     st.title("Idiomify Demo")
     text = st.text_area("Type sentences here",
-                        value="Just remember there will always be a hope even when things look black")
+                        value="Just remember that there will always be a hope even when things look hopeless")
     with st.sidebar:
         st.subheader("Supported idioms")
+        idioms = [row["Idiom"] for _, row in idioms.iterrows()]
         st.write(" / ".join(idioms))
 
     if st.button(label="Idiomify"):
         with st.spinner("Please wait..."):
             sents = [sent for sent in text.split(".") if sent]
-            sents = pipeline(sents, max_length=200)
+            preds = pipeline(sents, max_length=200)
             # highlight the rule & honorifics that were applied
-            st.write(". ".join(sents))
+            preds = [re.sub(r"<idiom>|</idiom>", "`", pred)
+                     for pred in preds]
+            st.markdown(". ".join(preds))
 
 
 if __name__ == '__main__':

main_eval.py

@@ -6,7 +6,7 @@ import pytorch_lightning as pl
 from pytorch_lightning.loggers import WandbLogger
 from transformers import BartTokenizer
 from idiomify.datamodules import IdiomifyDataModule
-from idiomify.fetchers import fetch_config, fetch_idiomifier
+from idiomify.fetchers import fetch_config, fetch_idiomifier, fetch_tokenizer
 from idiomify.paths import ROOT_DIR
 
 
@@ -17,10 +17,10 @@ def main():
     args = parser.parse_args()
     config = fetch_config()['idiomifier']
     config.update(vars(args))
-    tokenizer = BartTokenizer.from_pretrained(config['bart'])
     # prepare the datamodule
     with wandb.init(entity="eubinecto", project="idiomify", config=config) as run:
         model = fetch_idiomifier(config['ver'], run)  # fetch a pre-trained model
+        tokenizer = fetch_tokenizer(config['tokenizer_ver'], run)
         datamodule = IdiomifyDataModule(config, tokenizer, run)
         logger = WandbLogger(log_model=False)
         trainer = pl.Trainer(fast_dev_run=config['fast_dev_run'],

main_infer.py

@@ -3,25 +3,24 @@ This is for just a simple sanity check on the inference.
 """
 import argparse
 from idiomify.pipeline import Pipeline
-from idiomify.fetchers import fetch_config, fetch_idiomifier
+from idiomify.fetchers import fetch_config, fetch_idiomifier, fetch_tokenizer
 from transformers import BartTokenizer
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--sent", type=str,
-                        default="If there's any good to loosing my job,"
-                                " it's that I'll now be able to go to school full-time and finish my degree earlier.")
+                        default="Just remember that there will always be a hope even when things look hopeless")
     args = parser.parse_args()
     config = fetch_config()['idiomifier']
     config.update(vars(args))
     model = fetch_idiomifier(config['ver'])
+    tokenizer = fetch_tokenizer(config['tokenizer_ver'])
     model.eval()  # this is crucial
-    tokenizer = BartTokenizer.from_pretrained(config['bart'])
     pipeline = Pipeline(model, tokenizer)
     src = config['sent']
-    tgt = pipeline(sents=[config['sent']])
-    print(src, "\n->", tgt)
+    tgts = pipeline(sents=[src])
+    print(src, "\n->", tgts[0])
 
 
 if __name__ == '__main__':