Add gradio dataframes

README.md

@@ -6,6 +6,9 @@ colorTo: indigo
 sdk: gradio
 sdk_version: 4.26.0
 app_file: app.py
+fullWidth: true
+short_description: "cefrpy demo: Highlight English words according to CEFR scale"
+models: ["spacy/en_core_web_sm"]
 pinned: false
 license: mit
 ---

app.py

@@ -17,15 +17,28 @@ DEFAULT_ENTITY_ITEMS_TO_SKIP = [
     'ORG', 'FAC', 'PERSON'
 ]
 
+TOKEN_ATTRIBUTES = [
+    "Token",
+    "POS",
+    "Skipped",
+    "Level",
+    "Start",
+    "End"
+]
+
+WORDLIST_HEADER = ["Word", "Pos", "CEFR", "Level"]
+
+DEFAULT_WORDLIST_SLIDER_LEVEL = 4.0
+
 DEFAULT_TEXT = """The world's oldest known recipe is for beer. It dates back to around 5,000 BC and was found in ancient Sumeria (modern-day Iraq).
 
-Did you know that the shortest war in history lasted only 38 minutes? It occurred between Britain and Zanzibar on August 27, 1896. Zanzibar surrendered after the British issued an ultimatum to end their rule.
+Due to thermal expansion, the iron structure of the Eiffel Tower can expand in hot weather, making the tower grow by up to 6 inches (15 centimeters) in height.
 
-Some of the longest words in the English language can leave you tongue-tied! One example is "pneumonoultramicroscopicsilicovolcanoconiosis," which refers to a lung disease caused by inhaling very fine silica dust. With 45 letters, it's often cited as one of the longest words in English dictionaries. However, it's more of a curiosity than a practical term used in everyday language!
+Did you know that the word "antidisestablishmentarianism" is often cited as one of the longest non-technical words in the English language? It originated in the 19th century in Britain during debates over the disestablishment of the Church of England, and it refers to the opposition to the withdrawal of state support for an established church. This word has gained notoriety for its length and has been used as a challenge for spelling bees and word enthusiasts alike.
 
 In 2006, a Coca-Cola employee offered to sell Coca-Cola secrets to Pepsi. Pepsi responded by notifying Coca-Cola, and the FBI set up a sting operation to catch the culprit.
 
-Sir Isaac Newton, the renowned mathematician and physicist, invented the cat flap. While studying at Cambridge University, Newton had a pet cat named Spithead. He became annoyed when Spithead would interrupt his experiments, so he designed a small door in his study door through which the cat could come and go freely."""
+Like humans, cows form strong social bonds and often have "best friends" within their herds. They display complex social behaviors, including grooming, playing, and even grieving when separated from their friends."""
 
 DISPLACY_RENDER_OPTIONS = {
     "colors": {
@@ -52,8 +65,8 @@ ABBREVIATION_MAPPING = {
 
 LINKS_HTML = """
 <p>
-    &ensp;Github: <a href="https://github.com/Maximax67/cefrpy">https://github.com/Maximax67/cefrpy</a><br>
-    &ensp;Docs: <a href="https://maximax67.github.io/cefrpy">https://maximax67.github.io/cefrpy</a><br>
+    &ensp;Github: <a href="https://github.com/Maximax67/cefrpy">link</a><br>
+    &ensp;Docs: <a href="https://maximax67.github.io/cefrpy">link</a><br>
 </p>
 """
 
@@ -63,10 +76,6 @@ h1 {
     text-align: center;
     display:block;
 }
-
-.hide-container, .gr-group {
-    background: white !important;
-}
 """
 
 nlp = spacy.load(MODEL)
@@ -96,22 +105,58 @@ def get_dict_ents(text: str, tokens: list[tuple[str, str, bool, float, int, int]
     return dict_ents
 
 
-def render_visualization(text: str, ents_to_skip: list[str]) -> str:
+def get_cefr_tokens(text: str, ents_to_skip: list[str]) -> list[tuple[str, str, bool, float, int, int]]:
     doc = nlp(text)
-
     text_analyzer = CEFRSpaCyAnalyzer(entity_types_to_skip=ents_to_skip, abbreviation_mapping=ABBREVIATION_MAPPING)
     tokens = text_analyzer.analize_doc(doc)
+
+    return tokens
+
+
+def get_html_visualization(text: str, tokens: list[tuple[str, str, bool, float, int, int]]) -> str:
     dict_ents = get_dict_ents(text, tokens)
     html = displacy.render(dict_ents, manual=True, style="ent", options=DISPLACY_RENDER_OPTIONS)
 
     return html
 
 
+def get_wordlist_set(tokens: list[tuple[str, str, bool, float, int, int]],
+                min_level: float) -> set[tuple[str, str, bool, float, int, int]]:
+    filtered_tokens = set()
+    for word, pos, _, level, _, _ in tokens:
+        if level and level >= min_level:
+            filtered_tokens.add((word.lower(), pos, str(CEFRLevel(round(level))), level))
+
+    return filtered_tokens
+
+
+def get_wordlist(tokens: list[tuple[str, str, bool, float, int, int]], min_level: float):
+    wordlist_set = get_wordlist_set(tokens, min_level)
+    wordlist = list(wordlist_set)
+    wordlist.sort()
+
+    return wordlist
+
+
+def get_wordlist_from_dataframe(dataframe, min_level: float):
+    return get_wordlist(dataframe.values, min_level)
+
+
+def process_text(text: str, ents_to_skip: list[str] | None = DEFAULT_ENTITY_ITEMS_TO_SKIP, min_level: float = DEFAULT_WORDLIST_SLIDER_LEVEL) -> tuple[list[list], str]:
+    tokens = get_cefr_tokens(text, ents_to_skip)
+    html = get_html_visualization(text, tokens)
+    wordlist = get_wordlist(tokens, min_level)
+
+    return tokens, wordlist, html
+
+
+initial_tokens, initial_wordlist, initial_html = process_text(DEFAULT_TEXT)
+
 demo = gr.Blocks(css=CSS)
 
 with demo:
-    with gr.Row(variant="default"):
-        with gr.Group():
+    with gr.Row():
+        with gr.Column():
             with gr.Column():
                 with gr.Row():
                     gr.Markdown("# Gradio Demo: cefrpy")
@@ -141,19 +186,44 @@ with demo:
                         variant="primary"
                     )
 
-        with gr.Group():
+        with gr.Column():
             with gr.Row():
                 gr.Markdown("# Words CEFR level visualization")
 
             with gr.Row():
-                render_output = gr.HTML(
-                    value=render_visualization(DEFAULT_TEXT, DEFAULT_ENTITY_ITEMS_TO_SKIP),
+                rendered_html = gr.HTML(initial_html)
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                tokens_output = gr.Dataframe(headers=TOKEN_ATTRIBUTES, value=initial_tokens, interactive=False)
+
+        with gr.Column():
+            with gr.Row():
+                min_level_slider = gr.Slider(
+                    minimum=1.0,
+                    maximum=6.0, 
+                    value=DEFAULT_WORDLIST_SLIDER_LEVEL,
+                    step=0.02,
+                    interactive=True,
+                    label="Min level to generate word list"
                 )
 
+            with gr.Row():
+                wordlist = gr.Dataframe(headers=WORDLIST_HEADER, value=initial_wordlist, interactive=False)
+
     render_button.click(
-        render_visualization,
+        process_text,
         inputs=[text_input, ent_input],
-        outputs=render_output
+        outputs=[tokens_output, wordlist, rendered_html],
+        api_name="process_text"
+    )
+
+    min_level_slider.release(
+        get_wordlist_from_dataframe,
+        inputs=[tokens_output, min_level_slider],
+        outputs=[wordlist],
+        api_name=False
     )
 
-demo.launch()
+demo.launch(show_api=True)