Add tons of suggetsions; Improved matching logic

app.py

@@ -1,11 +1,16 @@
 import gradio as gr
 import huggingface_hub as hfh
 from requests.exceptions import HTTPError
+from functools import lru_cache
+
+from general_suggestions import GENERAL_SUGGESTIONS
+from model_suggestions import MODEL_SUGGESTIONS
+from task_suggestions import TASK_SUGGESTIONS
 
 # =====================================================================================================================
 # DATA
 # =====================================================================================================================
-# Dict with the tasks considered in this spaces, {pretty name: space tag}
+# Dict with the tasks considered in this spaces, {task: task tag}
 TASK_TYPES = {
     "✍️ Text Generation": "txtgen",
     "🤏 Summarization": "summ",
@@ -17,7 +22,7 @@ TASK_TYPES = {
     "🌇 Image to Text": "img2txt",
 }
 
-# Dict matching all task types with their possible hub tags, {space tag: (possible hub tags)}
+# Dict matching all task types with their possible hub tags, {task tag: (possible, hub, tags)}
 HUB_TAGS = {
     "txtgen": ("text-generation", "text2text-generation"),
     "summ": ("summarization", "text-generation", "text2text-generation"),
@@ -31,16 +36,16 @@ HUB_TAGS = {
 assert len(TASK_TYPES) == len(TASK_TYPES)
 assert all(tag in HUB_TAGS for tag in TASK_TYPES.values())
 
-# Dict with the problems considered in this spaces, {problem: space tag}
+# Dict with the problems considered in this spaces, {problem: problem tag}
 PROBLEMS = {
-    "🤔 Baseline. The model is returning nothing / random words": "baseline",
+    "🤔 Baseline. I'm getting gibberish and I want a baseline": "baseline",
     "😵 Crashes. I want to prevent my model from crashing again": "crashes",
-    "😵‍💫 Hallucinations. I would like to reduce them": "hallucinations",
-    "🌈 Interactivity. I would like a ChatGPT-like model": "interactity",
+    "🤥 Hallucinations. I would like to reduce them": "hallucinations",
     "📏 Length. I want to control the length of the output": "length",
-    "📈 Quality. I want to improve the overall quality": "quality",
+    "🌈 Prompting. I want better outputs without changing my generation options": "prompting",
+    "😵‍💫 Repetitions. Make them stop make them stop": "repetitions",
+    "📈 Quality. I want better outputs without changing my prompt": "quality",
     "🏎 Speed! Make it faster!": "speed",
-    "❓ ??? Something else, looking for ideas": "other",
 }
 
 INIT_MARKDOWN = """
@@ -68,9 +73,16 @@ DEMO_MARKDOWN = """
 """
 
 SUGGETIONS_HEADER = """
-✨ Here is a list of suggestions for you -- click to expand ✨
+#### ✨ Here is a list of suggestions for you -- click to expand ✨
 """
 
+PERFECT_MATCH_EMOJI = "✅"
+POSSIBLE_MATCH_EMOJI = "❓"
+MISSING_INPUTS = """
+💡 You can filter suggestions with {} if you add more inputs. Suggestions with {} are a perfect match.
+""".format(POSSIBLE_MATCH_EMOJI, PERFECT_MATCH_EMOJI)
+
+# The space below is reserved for suggestions that require advanced logic and/or formatting
 TASK_MODEL_MISMATCH = """
 <details><summary>{count}. Select a model better suited for your task.</summary>
 &nbsp;
@@ -89,94 +101,9 @@ models as a starting point. &nbsp;
 
 1. The tags of a model are defined by the community and are not always accurate. If you think the model is incorrectly
 tagged or missing a tag, please open an issue on the [model card](https://huggingface.co/{model_name}/tree/main).
-</details>
-"""
-
-SET_MAX_NEW_TOKENS = """
-<details><summary>{count}. Control the maximum output length with `max_new_tokens`.</summary>
-&nbsp;
-
-🤔 Why? &nbsp;
-
-All text generation calls have a length-related stopping condition. Depending on the model and/or the tool you're
-using to generate text, the default value may be too small or too large. I'd recommend ALWAYS setting this option.
-&nbsp;
-
-🤗 How? &nbsp;
-
-Our text generation interfaces accept a `max_new_tokens` option. Set it to define the maximum number of tokens
-that can be generated. &nbsp;
-
-😱 Caveats &nbsp;
-
-1. Allowing a longer output doesn't necessarily mean that the model will generate longer outputs. By default,
-the model will stop generating when it generates a special `eos_token_id` token.
-2. You shouldn't set `max_new_tokens` to a value larger than the maximum sequence length of the model. If you need a
-longer output, consider using a model with a larger maximum sequence length.
-3. The longer the output, the longer it will take to generate.
-</details>
-"""
-
-SET_MIN_LENGTH = """
-<details><summary>{count}. Force a minimum output length with `min_new_tokens`.</summary>
-&nbsp;
-
-🤔 Why? &nbsp;
 
-Text generation stops when the model generates a special `eos_token_id`. If you prevent it from happening, the model is
-forced to continue generating. &nbsp;
+_________________
 
-🤗 How? &nbsp;
-
-Our text generation interfaces accept a `min_new_tokens` argument. Set it to prevent `eos_token_id` from being
-generated until `min_new_tokens` tokens are generated. &nbsp;
-
-😱 Caveats &nbsp;
-
-1. The quality of the output may suffer if the model is forced to generate beyond its own original expectations.
-2. `min_new_tokens` must be smaller than than `max_new_tokens` (see related tip).
-</details>
-"""
-
-REMOVE_EOS_TOKEN = """
-<details><summary>{count}. Prevent the model of halting generation by removing `eos_token_id`.</summary>
-&nbsp;
-
-🤔 Why? &nbsp;
-
-Text generation stops when the model generates a special `eos_token_id`. If there is no `eos_token_id`, the model can't
-stop. &nbsp;
-
-
-🤗 How? &nbsp;
-
-Our text generation interfaces accept a `eos_token_id` argument. Set it to a null value (e.g., in Python,
-`eos_token_id=None`) to prevent generation to stop before it reaches other stopping conditions. &nbsp;
-
-😱 Caveats &nbsp;
-
-1. The quality of the output may suffer if the model is forced to generate beyond its own original expectations.
-</details>
-"""
-
-LIST_EOS_TOKEN = """
-<details><summary>{count}. Add a stop word through `eos_token_id`.</summary>
-&nbsp;
-
-🤔 Why? &nbsp;
-
-Text generation stops when the model generates a special `eos_token_id`. Actually, this attribute can be a list of
-tokens, which means you can define arbitrary stop words. &nbsp;
-
-
-🤗 How? &nbsp;
-
-Our text generation interfaces accept a `eos_token_id` argument. You can pass a list of tokens to make generation
-stop in the presence of any of those tokens. &nbsp;
-
-😱 Caveats &nbsp;
-
-1. When passing a list of tokens, you probably shouldn't forget to include the default `eos_token_id` there.
 </details>
 """
 # =====================================================================================================================
@@ -185,50 +112,88 @@ stop in the presence of any of those tokens. &nbsp;
 # =====================================================================================================================
 # SUGGESTIONS LOGIC
 # =====================================================================================================================
-def is_valid_task_for_model(model_name, task_type):
+def is_valid_task_for_model(model_tags, user_task):
+    if len(model_tags) == 0 or user_task == "":
+        return True  # No model / no tags = no problem :)
+
+    possible_tags = HUB_TAGS[user_task]
+    return any(tag in model_tags for tag in possible_tags)
+
+
+@lru_cache(maxsize=int(2e10))
+def get_model_tags(model_name):
     if model_name == "":
-        return True
+        return []
     try:
         model_tags = hfh.HfApi().model_info(model_name).tags
     except HTTPError:
-        return True  # Assume everything is okay
-
-    possible_tags = HUB_TAGS[TASK_TYPES[task_type]]
-    return any(tag in model_tags for tag in possible_tags)
+        model_tags = []
+    return model_tags
 
 
 def get_suggestions(task_type, model_name, problem_type):
     # Check if the inputs were given
-    if task_type == "" and model_name == "" and problem_type == "":
+    if all([task_type == "", model_name == "", problem_type == ""]):
         return INIT_MARKDOWN
 
-    suggestions = SUGGETIONS_HEADER
+    suggestions = ""
     counter = 0
+    model_tags = get_model_tags(model_name)
+
+    user_problem = PROBLEMS.get(problem_type, "")
+    user_task = TASK_TYPES.get(task_type, "")
 
     # Check if the model is valid for the task. If not, return straight away
-    if not is_valid_task_for_model(model_name, task_type):
+    if not is_valid_task_for_model(model_tags, user_task):
         counter += 1
-        possible_tags = " ".join("`" + tag + "`" for tag in HUB_TAGS[TASK_TYPES[task_type]])
+        possible_tags = " ".join("`" + tag + "`" for tag in HUB_TAGS[user_task])
         suggestions += TASK_MODEL_MISMATCH.format(
-            count=counter, model_name=model_name, task_type=task_type, tags=possible_tags
+            count=counter, model_name=model_name, task_type=user_task, tags=possible_tags
         )
         return suggestions
 
     # Demo shortcut: only a few sections are working
-    if PROBLEMS.get(problem_type, "") not in ("", "length", "quality", "speed"):
+    if user_problem not in ("", "length", "quality"):
         return DEMO_MARKDOWN
 
-    if PROBLEMS.get(problem_type, "") == "length":
-        counter += 1
-        suggestions += SET_MAX_NEW_TOKENS.format(count=counter)
-        counter += 1
-        suggestions += SET_MIN_LENGTH.format(count=counter)
-        counter += 1
-        suggestions += REMOVE_EOS_TOKEN.format(count=counter)
-        counter += 1
-        suggestions += LIST_EOS_TOKEN.format(count=counter)
-
-    return suggestions
+    # First: model-specific suggestions
+    has_model_specific_suggestions = False
+    match_emoji = POSSIBLE_MATCH_EMOJI if (user_problem == "" or len(model_tags) == 0) else PERFECT_MATCH_EMOJI
+    for model_tag, problem_tags, suggestion in MODEL_SUGGESTIONS:
+        if user_problem == "" or user_problem in problem_tags:
+            if len(model_tags) == 0 or model_tag in model_tags:
+                counter += 1
+                suggestions += suggestion.format(count=counter, match_emoji=match_emoji)
+                has_model_specific_suggestions = True
+
+    # Second: task-specific suggestions
+    has_task_specific_suggestions = False
+    match_emoji = POSSIBLE_MATCH_EMOJI if (user_problem == "" or user_task == "") else PERFECT_MATCH_EMOJI
+    for task_tags, problem_tags, suggestion in TASK_SUGGESTIONS:
+        if user_problem == "" or user_problem in problem_tags:
+            if user_task == "" or user_task in task_tags:
+                counter += 1
+                suggestions += suggestion.format(count=counter, match_emoji=match_emoji)
+                has_task_specific_suggestions = True
+
+    # Finally: general suggestions for the problem
+    has_problem_specific_suggestions = False
+    match_emoji = POSSIBLE_MATCH_EMOJI if user_problem == "" else PERFECT_MATCH_EMOJI
+    for problem_tags, suggestion in GENERAL_SUGGESTIONS:
+        if user_problem == "" or user_problem in problem_tags:
+            counter += 1
+            suggestions += suggestion.format(count=counter, match_emoji=match_emoji)
+            has_problem_specific_suggestions = True
+
+    # Prepends needed bits
+    if (
+        (task_type == "" and has_task_specific_suggestions)
+        or (model_name == "" and has_model_specific_suggestions)
+        or (problem_type == "" and has_problem_specific_suggestions)
+    ):
+        suggestions = MISSING_INPUTS + suggestions
+
+    return SUGGETIONS_HEADER + suggestions
 # =====================================================================================================================
 
 
@@ -255,7 +220,7 @@ with demo:
                 value="",
             )
             task_type = gr.Dropdown(
-                label="What task are you working on?",
+                label="Which task are you working on?",
                 choices=[""] + list(TASK_TYPES.keys()),
                 interactive=True,
                 value="",

general_suggestions.py

@@ -0,0 +1,156 @@
+"""
+This is a file holding task and model agnostic suggestions.
+
+How to add a new suggestion:
+1. Add a new constant at the bottom of the file with your suggestion. Please try to follow the same format as the
+existing suggestions.
+2. Add a new entry to the `GENERAL_SUGGESTIONS`, with format `((problem tags,), suggestion constant)`.
+    a. See `app.py` for the existing problem tags.
+    c. Make sure the problem tags are a tuple.
+"""
+
+SET_MAX_NEW_TOKENS = """
+<details><summary>{match_emoji} {count}. Control the maximum output length.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+All text generation calls have a length-related stopping condition. Depending on the model and/or the tool you're
+using to generate text, the default value may be too small or too large. I'd recommend ALWAYS setting this option.
+&nbsp;
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `max_new_tokens` option. Set it to define the maximum number of tokens
+that can be generated. &nbsp;
+
+😱 Caveats &nbsp;
+
+1. Allowing a longer output doesn't necessarily mean that the model will generate longer outputs. By default,
+the model will stop generating when it generates a special `eos_token_id` token.
+2. You shouldn't set `max_new_tokens` to a value larger than the maximum sequence length of the model. If you need a
+longer output, consider using a model with a larger maximum sequence length.
+3. The longer the output, the longer it will take to generate.
+_________________
+</details>
+"""
+
+SET_MIN_LENGTH = """
+<details><summary>{match_emoji} {count}. Force a minimum output length.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+Text generation stops when the model generates a special `eos_token_id`. If you prevent it from happening, the model is
+forced to continue generating. &nbsp;
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `min_new_tokens` argument. Set it to prevent `eos_token_id` from being
+generated until `min_new_tokens` tokens are generated. &nbsp;
+
+😱 Caveats &nbsp;
+
+1. The quality of the output may suffer if the model is forced to generate beyond its own original expectations.
+2. `min_new_tokens` must be smaller than than `max_new_tokens` (see related tip).
+_________________
+</details>
+"""
+
+REMOVE_EOS_TOKEN = """
+<details><summary>{match_emoji} {count}. Force the model to generate until it reaches the maximum output length.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+Text generation stops when the model generates a special `eos_token_id`. If there is no `eos_token_id`, the model can't
+stop. &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `eos_token_id` argument. Set it to a null value (e.g., in Python,
+`eos_token_id=None`) to prevent generation to stop before it reaches other stopping conditions. &nbsp;
+
+😱 Caveats &nbsp;
+
+1. The quality of the output may suffer if the model is forced to generate beyond its own original expectations.
+_________________
+</details>
+"""
+
+LIST_EOS_TOKEN = """
+<details><summary>{match_emoji} {count}. Add a stop word.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+Text generation stops when the model generates a special `eos_token_id`. Actually, this attribute can be a list of
+tokens, which means you can define arbitrary stop words. &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `eos_token_id` argument. You can pass a list of tokens to make generation
+stop in the presence of any of those tokens. &nbsp;
+
+😱 Caveats &nbsp;
+
+1. When passing a list of tokens, you probably shouldn't forget to include the default `eos_token_id` there.
+_________________
+</details>
+"""
+
+TRY_CONTRASTIVE_SEARCH = """
+<details><summary>{match_emoji} {count}. Try Contrastive Search.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+Contrastive Search is a greedy decoding strategy that strikes a balance between picking the best token and avoiding
+repetition in the representation space. Despite being a greedy decoding strategy, it can also perform well on tasks
+that require creativity (i.e. Sampling territory). In some models, it greatly reduces the problem of repetition. &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept two arguments: `top_k` and `penalty_alpha`. The authors recomment starting with
+`top_k=4` and `penalty_alpha=0.6`. &nbsp;
+
+😱 Caveats &nbsp;
+
+1. Contrastive Search does not work well with all models -- it depends on how distributed their representation spaces
+are. See [this thread](https://huggingface.co/spaces/joaogante/contrastive_search_generation/discussions/1#63764a108623a4a7954a5be5)
+for further information.
+_________________
+</details>
+"""
+
+BLOCK_BAD_WORDS = """
+<details><summary>{match_emoji} {count}. Prevent certain words from being generated.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+You might want to prevent your model from generating certain tokens, such as swear words. &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `bad_words_ids` argument. There, you can pass a list of lists, where each
+inner list contains a forbidden sequence of tokens.
+Remember that you can get the token IDs for the words you want to block through
+`bad_word_ids = tokenizer(bad_words, add_prefix_space=True, add_special_tokens=False).input_ids` &nbsp;
+_________________
+</details>
+"""
+
+GENERAL_SUGGESTIONS = (
+    (("length",), SET_MAX_NEW_TOKENS),
+    (("length",), SET_MIN_LENGTH),
+    (("length",), REMOVE_EOS_TOKEN),
+    (("length",), LIST_EOS_TOKEN),
+    (("quality", "repetitions"), TRY_CONTRASTIVE_SEARCH),
+    (("quality",), BLOCK_BAD_WORDS),
+)
+assert all(isinstance(problem_tags, tuple) for problem_tags, _ in GENERAL_SUGGESTIONS)

model_suggestions.py

@@ -0,0 +1,62 @@
+"""
+This is a file holding model-specific suggestions.
+
+How to add a new suggestion:
+1. Add a new constant at the bottom of the file with your suggestion. Please try to follow the same format as the
+existing suggestions.
+2. Add a new entry to the `MODEL_SUGGESTIONS`, with format `(model tag, (problem tags,), suggestion constant)`.
+    a. Make sure the model tag matches the exact same tag as on the Hub (e.g. GPT-J is `gptj`)
+    b. See `app.py` for the existing problem tags.
+    c. Make sure the problem tags are a tuple.
+"""
+
+
+GPTJ_USE_SAMPLING = """
+<details><summary>{match_emoji} {count}. GPT-J - Avoid using Greedy Search and Beam Search.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+According to its creators, "generating without sampling was actually surprisingly suboptimal". &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `do_sample` argument. Set it to `True` to ensure sampling-based strategies
+are used. &nbsp;
+
+💡 Source &nbsp;
+
+1. [This tweet](https://twitter.com/EricHallahan/status/1627785461723721729) by a core member of EleutherAI, the
+creator of GPT-J
+_________________
+</details>
+"""
+
+
+T5_FLOAT16 = """
+<details><summary>{match_emoji} {count}. T5 - If you're using int8 or float16, make sure you have `transformers>=4.26.1`.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+In a nutshell, some layers in T5 don't work well in lower precision unless they are in bf16. Newer versions of
+`transformers` take care of upcasting the layers when needed. &nbsp;
+
+
+🤗 How? &nbsp;
+
+Make sure the dependencies in your workflow have `transformers>=4.26.1` &nbsp;
+
+💡 Source &nbsp;
+
+1. See [this thread](https://github.com/huggingface/transformers/issues/20287) for the full discussion.
+_________________
+</details>
+"""
+
+MODEL_SUGGESTIONS = (
+    ("gptj", ("quality",), GPTJ_USE_SAMPLING),
+    ("t5", ("quality", "baseline", "speed"), T5_FLOAT16),
+)
+assert all(isinstance(problem_tags, tuple) for _, problem_tags, _ in MODEL_SUGGESTIONS)

task_suggestions.py

@@ -0,0 +1,85 @@
+"""
+This is a file holding task-specific suggestions.
+
+How to add a new suggestion:
+1. Add a new constant at the bottom of the file with your suggestion. Please try to follow the same format as the
+existing suggestions.
+2. Add a new entry to the `TASK_SUGGESTIONS`, with format `((task tags,), (problem tag,), suggestion constant)`.
+    a. See `app.py` for the existing task tags.
+    b. See `app.py` for the existing problem tags.
+    c. Make sure the task tags and the problem tags are a tuple.
+"""
+
+USE_SAMPLING = """
+<details><summary>{match_emoji} {count}. Use sampling decoding strategies.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+The selected task benefits from creativity. Sampling-based decoding strategies typically yield better results in
+creativity-based tasks &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `do_sample` argument. Set it to `True` to ensure sampling-based strategies
+are used. &nbsp;
+_________________
+</details>
+"""
+
+
+BLOCK_LOW_PROBA_TOKENS = """
+<details><summary>{match_emoji} {count}. Sampling - Block low probability tokens.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+When decoding with sampling-based strategies, ANY token in the model vocabulary can be selected. This means there is
+always a chance that the generated text drifts off-topic. &nbsp;
+
+
+🤗 How? &nbsp;
+
+There are a few different strategies you can try. They all discard (i.e. set probability to 0) tokens at each
+generation step. Unless stated otherwise, they can be used with each other.
+1. Top K: discards all but the K most likely tokens (suggeted value: `top_k=50`);
+2. Top P: sorts tokens by probability in descending order, computes the cumulative probability, then discards all
+tokens with a cumulative probability above the threshold P (suggested value: `top_p=0.95`);
+3. ETA Cutoff: A balance between Top K and Top P. See the [corresponding paper](https://arxiv.org/abs/2210.15191) for
+more details (should not be used with others; suggested value: `eta_cutoff=1e-3`). &nbsp;
+_________________
+</details>
+"""
+
+
+USE_DETERMINISTIC = """
+<details><summary>{match_emoji} {count}. Use greedy decoding strategies.</summary>
+&nbsp;
+
+🤔 Why? &nbsp;
+
+The selected task is factual, it does not benefit from creativity. Greedy decoding strategies (like Greedy Search and
+Beam Search) are preferred in those situations. &nbsp;
+
+
+🤗 How? &nbsp;
+
+Our text generation interfaces accept a `do_sample` argument. Set it to `False` to ensure greedy strategies
+are used. &nbsp;
+_________________
+</details>
+"""
+
+# task tags that should use sampling and benefit from sampling-related advice
+sampling = ("txtgen", "chat", "img2txt")
+# task tags that should NOT use sampling and benefit from greedy/beam search advice
+greedy = ("summ", "trans", "txtqa", "otherqa", "asr")
+
+TASK_SUGGESTIONS = (
+    (sampling, ("quality",), USE_SAMPLING),
+    (sampling, ("quality", "hallucinations"), BLOCK_LOW_PROBA_TOKENS),
+    (greedy, ("quality",), USE_DETERMINISTIC),
+)
+assert all(isinstance(problem_tags, tuple) for _, problem_tags, _ in TASK_SUGGESTIONS)
+assert all(isinstance(task_tags, tuple) for task_tags, _, _ in TASK_SUGGESTIONS)