add GEM section

datacards/gem.py

@@ -4,10 +4,73 @@ from .streamlit_utils import (
     make_text_input
 )
 
-N_FIELDS = 1
+from .streamlit_utils import (
+    make_multiselect,
+    make_selectbox,
+    make_text_area,
+    make_text_input,
+    make_radio,
+)
+
+N_FIELDS_RATIONALE = 5
+N_FIELDS_STARTING = 2
+
+N_FIELDS = N_FIELDS_RATIONALE + \
+    N_FIELDS_STARTING
+
 
 def gem_page():
-    return None
+    st.session_state.card_dict["gem"] = st.session_state.card_dict.get("gem", {})
+    with st.expander("Rationale", expanded=False):
+        key_pref = ["gem", "rationale"]
+        st.session_state.card_dict["gem"]["rationale"] = st.session_state.card_dict["gem"].get("rationale", {})
+        make_text_area(
+            label="Contribution toward better generation evaluation",
+            key_list=key_pref + ["contribution"],
+            help="Describe briefly what makes this dataset an interesting target for NLG evaluations and why it is part of GEM",
+        )
+        make_radio(
+            label="Do other datasets for the high level task exist?",
+            options=["no", "yes"],
+            key_list=key_pref + ["sole-task-dataset"],
+            help="for example, is this the only summarization dataset proposed in GEM",
+        )
+        make_radio(
+            label="Does this dataset cover other languages than other datasets for the same task?",
+            options=["no", "yes"],
+            key_list=key_pref + ["sole-language-task-dataset"],
+            help="for example, is this the only summarization dataset proposed in GEM to have French text?",
+        )
+        make_text_area(
+            label="What else sets this dataset apart from other similar datasets in GEM?",
+            key_list=key_pref + ["distinction-description"],
+            help="Describe briefly for each similar dataset (same task/languages) what sets this one apart",
+        )
+        make_text_area(
+            label="What aspect of model ability can be measured with this dataset?",
+            key_list=key_pref + ["model-ability"],
+            help="What kind of abilities should a model exhibit that performs well on the task of this dataset (e.g., reasoning capability, morphological inflection)?",
+        )
+    with st.expander("Getting Started", expanded=False):
+        key_pref = ["gem", "starting"]
+        st.session_state.card_dict["gem"]["starting"] = st.session_state.card_dict["gem"].get("starting", {})
+        make_text_area(
+            label="Getting started with in-depth research on the task. Add relevant pointers to resources that researchers can consult when they want to get started digging deeper into the task.",
+            key_list=key_pref + ["research-pointers"],
+            help=" These can include blog posts, research papers, literature surveys, etc. You can also link to tutorials on the GEM website.",
+        )
+        make_text_area(
+            label="Technical terms used in this card and the dataset and their definitions",
+            key_list=key_pref + ["technical-terms"],
+            help="Provide a brief definition of technical terms that are unique to this dataset",
+        )
+
 
 def gem_summary():
-    return None
+    total_filled = sum([len(dct) for dct in st.session_state.card_dict.get('gem', {}).values()])
+    with st.expander(f"Dataset in GEM Completion - {total_filled} of {N_FIELDS}", expanded=False):
+        completion_markdown = ""
+        completion_markdown += f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
+        completion_markdown += f"- **Sub-section - Rationale:**\n  - {len(st.session_state.card_dict.get('gem', {}).get('rationale', {}))} of {N_FIELDS_RATIONALE} fields\n"
+        completion_markdown += f"- **Sub-section - Getting Started:**\n  - {len(st.session_state.card_dict.get('gem', {}).get('starting', {}))} of {N_FIELDS_STARTING} fields\n"
+        st.markdown(completion_markdown)

datacards/overview.py

@@ -34,7 +34,7 @@ def overview_page():
     st.session_state.card_dict["overview"] = st.session_state.card_dict.get("overview", {})
     with st.expander("Where to find", expanded=False):
         key_pref = ["overview", "where"]
-        st.session_state.card_dict["overview"]["where"] = st.session_state.card_dict.get("where", {})
+        st.session_state.card_dict["overview"]["where"] = st.session_state.card_dict["overview"].get("where", {})
         make_text_input(
             label="What is the webpage for the dataset (if it exists)?",
             key_list=key_pref + ["website"],
@@ -83,7 +83,7 @@ def overview_page():
         )
     with st.expander("Languages and Intended Use", expanded=False):
         key_pref = ["overview", "languages"]
-        st.session_state.card_dict["overview"]["languages"] = st.session_state.card_dict.get("languages", {})
+        st.session_state.card_dict["overview"]["languages"] = st.session_state.card_dict["overview"].get("languages", {})
         make_radio(
             label="Is the dataset multilingual?",
             options=["no", "yes"],
@@ -184,9 +184,10 @@ def overview_page():
 
 
 def overview_summary():
-    with st.expander("Dataset Overview Completion", expanded=True):
+    total_filled = sum([len(dct) for dct in st.session_state.card_dict.get('overview', {}).values()])
+    with st.expander(f"Dataset Overview Completion - {total_filled} of {N_FIELDS}", expanded=False):
         completion_markdown = ""
-        completion_markdown += f"- **Overall competion:**\n  - {sum([len(dct) for dct in st.session_state.card_dict.get('overview', {}).values()])} of {N_FIELDS} fields\n"
+        completion_markdown += f"- **Overall competion:**\n  - {total_filled} of {N_FIELDS} fields\n"
         completion_markdown += f"- **Sub-section - Where to find:**\n  - {len(st.session_state.card_dict.get('overview', {}).get('where', {}))} of {N_FIELDS_WHERE} fields\n"
         completion_markdown += f"- **Sub-section - Languages and Intended Use:**\n  - {len(st.session_state.card_dict.get('overview', {}).get('languages', {}))} of {N_FIELDS_LANGUAGES} fields\n"
         completion_markdown += f"- **Sub-section - Credit:**\n  - {len(st.session_state.card_dict.get('overview', {}).get('credit', {}))} of {N_FIELDS_CREDIT} fields\n"

datacards/streamlit_utils.py

@@ -2,15 +2,20 @@ import streamlit as st
 
 
 # Streamlit widgets with persistence
-# TODO: better detection of whether a field has been updated
-def update_card_dict(key_list, default=None):
+def is_filled(key_list):
+    state_filled_key = "_".join(key_list) + "_filled"
+    def on_change_action():
+        st.session_state.save_state[state_filled_key] = True
+    return on_change_action
+
+def update_card_dict(key_list, use_default=None):
     state_key = "_".join(key_list)
-    if st.session_state.save_state.get(state_key, default) != default:
+    if st.session_state.save_state.get(state_key + "_filled", False) or use_default:
         card_key = key_list[-1]
         current_dict = st.session_state.card_dict
         for key in key_list[:-1]:
             current_dict = current_dict[key]
-        current_dict[card_key] = st.session_state.save_state[state_key]
+        current_dict[card_key] = st.session_state.save_state.get(state_key, use_default)
 
 
 def make_multiselect(
@@ -27,13 +32,14 @@ def make_multiselect(
         format_func=format_func,
         key=key,
         default=st.session_state.save_state.get(key, []),
+        on_change=is_filled(key_list),
         help=help,
     )
-    update_card_dict(key_list, default=[])
+    update_card_dict(key_list)
     return res
 
 
-def make_selectbox(key_list, label, options, format_func=lambda x: x, help="", index=None, on_change=None):
+def make_selectbox(key_list, label, options, format_func=lambda x: x, help="", index=None):
     key = "_".join(key_list)
     if key in st.session_state:
         st.session_state.save_state[key] = st.session_state[key]
@@ -47,10 +53,10 @@ def make_selectbox(key_list, label, options, format_func=lambda x: x, help="", i
         index=options.index(
             st.session_state.save_state.get(key, options[0])
         ),  # if st.session_state.save_state.get(key, options[0]) in options else 0,
+        on_change=is_filled(key_list),
         help=help,
-        on_change=on_change,
     )
-    update_card_dict(key_list, default=[])
+    update_card_dict(key_list, use_default=st.session_state.save_state.get(key, options[0]))  # use the default value even without interactions
     return res
 
 
@@ -66,9 +72,10 @@ def make_radio(key_list, label, options, format_func=lambda x: x, help="", index
         format_func=format_func,
         key=key,
         index=options.index(st.session_state.save_state.get(key, options[0])),
+        on_change=is_filled(key_list),
         help=help,
     )
-    update_card_dict(key_list)
+    update_card_dict(key_list, use_default=st.session_state.save_state.get(key, options[0]))  # use the default value even without interactions
     return res
 
 
@@ -82,9 +89,10 @@ def make_text_input(key_list, label, help="", value=None):
         label=label,
         key=key,
         value=st.session_state.save_state.get(key, ""),
+        on_change=is_filled(key_list),
         help=help,
     )
-    update_card_dict(key_list, default="")
+    update_card_dict(key_list)
     return res
 
 
@@ -98,9 +106,10 @@ def make_text_area(key_list, label, help="", value=None):
         label=label,
         key=key,
         value=st.session_state.save_state.get(key, ""),
+        on_change=is_filled(key_list),
         help=help,
     )
-    update_card_dict(key_list, default="")
+    update_card_dict(key_list)
     return res
 
 
@@ -114,6 +123,7 @@ def make_checkbox(key_list, label, help="", value=None):
         label=label,
         key=key,
         value=st.session_state.save_state.get(key, False),
+        on_change=is_filled(key_list),
         help=help,
     )
     update_card_dict(key_list)