Repository: RasaHQ/rasa

Issue:
`rasa shell` does not allow free text input when template has buttons in it
<!-- THIS INFORMATION IS MANDATORY - YOUR ISSUE WILL BE CLOSED IF IT IS MISSING. If you don't know your Rasa version, use `rasa --version`. 
Please format any code or console output with three ticks ``` above and below.
If you are asking a usage question (e.g. "How do I do xyz") please post your question on https://forum.rasa.com instead -->

**Rasa version**: 1.1.8

**Issue**:
The bot should accept both free text alongside the choice of one of the buttons (since that'd be possible in Rasa X and presumably other channels as well), but currently there's no way to do that in `rasa shell`.


Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 926952497b0..f42e1713c5d 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -23,7 +23,8 @@ Removed
 
 Fixed
 -----
-
+- Free text input was not allowed in the Rasa shell when the response template contained buttons,
+  which has now been fixed.
 
 [1.2.2] - 2019-08-07
 ^^^^^^^^^^^^^^^^^^^^
diff --git a/rasa/cli/utils.py b/rasa/cli/utils.py
index 2158d6b954a..74542b82d4c 100644
--- a/rasa/cli/utils.py
+++ b/rasa/cli/utils.py
@@ -1,6 +1,8 @@
 import os
 import sys
-from typing import Any, Optional, Text, List
+import json
+import re
+from typing import Any, Optional, Text, List, Dict
 import logging
 from questionary import Question
 
@@ -9,6 +11,9 @@ from rasa.constants import DEFAULT_MODELS_PATH
 logger = logging.getLogger(__name__)
 
 
+FREE_TEXT_INPUT_PROMPT = "Type out your own message..."
+
+
 def get_validated_path(
     current: Optional[Text],
     parameter: Text,
@@ -124,6 +129,67 @@ def create_output_path(
         return os.path.join(output_path, file_name)
 
 
+def button_to_string(button: Dict[Text, Any], idx: int = 0) -> Text:
+    """Create a string representation of a button."""
+
+    title = button.pop("title", "")
+
+    if "payload" in button:
+        payload = " ({})".format(button.pop("payload"))
+    else:
+        payload = ""
+
+    # if there are any additional attributes, we append them to the output
+    if button:
+        details = " - {}".format(json.dumps(button, sort_keys=True))
+    else:
+        details = ""
+
+    button_string = "{idx}: {title}{payload}{details}".format(
+        idx=idx + 1, title=title, payload=payload, details=details
+    )
+
+    return button_string
+
+
+def element_to_string(element: Dict[Text, Any], idx: int = 0) -> Text:
+    """Create a string representation of an element."""
+    title = element.pop("title", "")
+
+    element_string = "{idx}: {title} - {element}".format(
+        idx=idx + 1, title=title, element=json.dumps(element, sort_keys=True)
+    )
+
+    return element_string
+
+
+def button_choices_from_message_data(
+    message: Dict[Text, Any], allow_free_text_input: bool = True
+) -> Question:
+    """Return list of choices to present to the user.
+
+    If allow_free_text_input is True, an additional option is added
+    at the end along with the template buttons that allows the user
+    to type in free text.
+    """
+    choices = [
+        button_to_string(button, idx)
+        for idx, button in enumerate(message.get("buttons"))
+    ]
+    if allow_free_text_input:
+        choices.append(FREE_TEXT_INPUT_PROMPT)
+    return choices
+
+
+def payload_from_button_question(button_question: Question) -> Text:
+    """Prompt user with a button question and returns the nlu payload."""
+    response = button_question.ask()
+    if response != FREE_TEXT_INPUT_PROMPT:
+        # Extract intent slash command if it's a button
+        response = response[response.find("(") + 1 : response.find(")")]
+    return response
+
+
 class bcolors(object):
     HEADER = "\033[95m"
     OKBLUE = "\033[94m"
@@ -162,11 +228,3 @@ def print_error(*args: Any):
 def signal_handler(sig, frame):
     print ("Goodbye 👋")
     sys.exit(0)
-
-
-def payload_from_button_question(button_question: Question) -> Text:
-    """Prompts user with a button question and returns the nlu payload."""
-    response = button_question.ask()
-    payload = response[response.find("(") + 1 : response.find(")")]
-
-    return payload
diff --git a/rasa/core/channels/channel.py b/rasa/core/channels/channel.py
index 7d494a905e6..c10ff188437 100644
--- a/rasa/core/channels/channel.py
+++ b/rasa/core/channels/channel.py
@@ -9,6 +9,7 @@ from sanic.request import Request
 from typing import Text, List, Dict, Any, Optional, Callable, Iterable, Awaitable
 
 import rasa.utils.endpoints
+from rasa.cli import utils as cli_utils
 from rasa.constants import DOCS_BASE_URL
 from rasa.core import utils
 
@@ -89,41 +90,6 @@ def register(
     app.input_channels = input_channels
 
 
-def button_to_string(button, idx=0):
-    """Create a string representation of a button."""
-
-    title = button.pop("title", "")
-
-    if "payload" in button:
-        payload = " ({})".format(button.pop("payload"))
-    else:
-        payload = ""
-
-    # if there are any additional attributes, we append them to the output
-    if button:
-        details = " - {}".format(json.dumps(button, sort_keys=True))
-    else:
-        details = ""
-
-    button_string = "{idx}: {title}{payload}{details}".format(
-        idx=idx + 1, title=title, payload=payload, details=details
-    )
-
-    return button_string
-
-
-def element_to_string(element, idx=0):
-    """Create a string representation of an element."""
-
-    title = element.pop("title", "")
-
-    element_string = "{idx}: {title} - {element}".format(
-        idx=idx + 1, title=title, element=json.dumps(element, sort_keys=True)
-    )
-
-    return element_string
-
-
 class InputChannel(object):
     @classmethod
     def name(cls):
@@ -255,7 +221,7 @@ class OutputChannel(object):
 
         await self.send_text_message(recipient_id, text, **kwargs)
         for idx, button in enumerate(buttons):
-            button_msg = button_to_string(button, idx)
+            button_msg = cli_utils.button_to_string(button, idx)
             await self.send_text_message(recipient_id, button_msg, **kwargs)
 
     async def send_quick_replies(
diff --git a/rasa/core/channels/console.py b/rasa/core/channels/console.py
index 65c110159d7..da2377680ae 100644
--- a/rasa/core/channels/console.py
+++ b/rasa/core/channels/console.py
@@ -9,37 +9,36 @@ from aiohttp import ClientTimeout
 from async_generator import async_generator, yield_
 from prompt_toolkit.styles import Style
 
-import rasa.cli.utils
+from rasa.cli import utils as cli_utils
+
 from rasa.core import utils
 from rasa.core.channels.channel import UserMessage
-from rasa.core.channels.channel import RestInput, button_to_string, element_to_string
+from rasa.core.channels.channel import RestInput
 from rasa.core.constants import DEFAULT_SERVER_URL
 from rasa.core.interpreter import INTENT_MESSAGE_PREFIX
 
 logger = logging.getLogger(__name__)
 
+
 DEFAULT_STREAM_READING_TIMEOUT_IN_SECONDS = 10
 
 
 def print_bot_output(
-    message, color=rasa.cli.utils.bcolors.OKBLUE
+    message, color=cli_utils.bcolors.OKBLUE
 ) -> Optional[questionary.Question]:
     if ("text" in message) and not ("buttons" in message):
-        rasa.cli.utils.print_color(message.get("text"), color=color)
+        cli_utils.print_color(message.get("text"), color=color)
 
     if "image" in message:
-        rasa.cli.utils.print_color("Image: " + message.get("image"), color=color)
+        cli_utils.print_color("Image: " + message.get("image"), color=color)
 
     if "attachment" in message:
-        rasa.cli.utils.print_color(
-            "Attachment: " + message.get("attachment"), color=color
-        )
+        cli_utils.print_color("Attachment: " + message.get("attachment"), color=color)
 
     if "buttons" in message:
-        choices = [
-            button_to_string(button, idx)
-            for idx, button in enumerate(message.get("buttons"))
-        ]
+        choices = cli_utils.button_choices_from_message_data(
+            message, allow_free_text_input=True
+        )
 
         question = questionary.select(
             message.get("text"),
@@ -49,32 +48,34 @@ def print_bot_output(
         return question
 
     if "elements" in message:
-        rasa.cli.utils.print_color("Elements:", color=color)
+        cli_utils.print_color("Elements:", color=color)
         for idx, element in enumerate(message.get("elements")):
-            rasa.cli.utils.print_color(element_to_string(element, idx), color=color)
+            cli_utils.print_color(
+                cli_utils.element_to_string(element, idx), color=color
+            )
 
     if "quick_replies" in message:
-        rasa.cli.utils.print_color("Quick Replies:", color=color)
+        cli_utils.print_color("Quick Replies:", color=color)
         for idx, element in enumerate(message.get("quick_replies")):
-            rasa.cli.utils.print_color(button_to_string(element, idx), color=color)
+            cli_utils.print_color(cli_utils.button_to_string(element, idx), color=color)
 
     if "custom" in message:
-        rasa.cli.utils.print_color("Custom json:", color=color)
-        rasa.cli.utils.print_color(
-            json.dumps(message.get("custom"), indent=2), color=color
-        )
+        cli_utils.print_color("Custom json:", color=color)
+        cli_utils.print_color(json.dumps(message.get("custom"), indent=2), color=color)
 
 
-def get_cmd_input(button_question: questionary.Question) -> Optional[Text]:
+def get_user_input(button_question: questionary.Question) -> Optional[Text]:
     if button_question is not None:
-        response = rasa.cli.utils.payload_from_button_question(button_question)
+        response = cli_utils.payload_from_button_question(button_question)
+        if response == cli_utils.FREE_TEXT_INPUT_PROMPT:
+            # Re-prompt user with a free text input
+            response = get_user_input(None)
     else:
         response = questionary.text(
             "",
             qmark="Your input ->",
             style=Style([("qmark", "#b373d6"), ("", "#b373d6")]),
         ).ask()
-
     return response.strip() if response is not None else None
 
 
@@ -117,7 +118,7 @@ async def record_messages(
 
     exit_text = INTENT_MESSAGE_PREFIX + "stop"
 
-    rasa.cli.utils.print_success(
+    cli_utils.print_success(
         "Bot loaded. Type a message and press enter "
         "(use '{}' to exit): ".format(exit_text)
     )
@@ -125,7 +126,7 @@ async def record_messages(
     num_messages = 0
     button_question = None
     while not utils.is_limit_reached(num_messages, max_message_limit):
-        text = get_cmd_input(button_question)
+        text = get_user_input(button_question)
 
         if text == exit_text or text is None:
             break
diff --git a/rasa/core/restore.py b/rasa/core/restore.py
index c85c6436c74..145a3a50829 100644
--- a/rasa/core/restore.py
+++ b/rasa/core/restore.py
@@ -8,7 +8,7 @@ import rasa.utils.io
 import typing
 from typing import List, Text, Tuple
 
-from rasa.cli import utils as cliutils
+from rasa.cli import utils as cli_utils
 from rasa.core.actions.action import ACTION_LISTEN_NAME
 from rasa.core.channels import console
 from rasa.core.channels.channel import UserMessage, CollectingOutputChannel
@@ -93,7 +93,7 @@ async def replay_events(tracker: DialogueStateTracker, agent: "Agent") -> None:
             )
 
             actions_between_utterances = []
-            cliutils.print_success(event.text)
+            cli_utils.print_success(event.text)
             out = CollectingOutputChannel()
             await agent.handle_text(
                 event.text, sender_id=tracker.sender_id, output_channel=out
@@ -107,7 +107,7 @@ async def replay_events(tracker: DialogueStateTracker, agent: "Agent") -> None:
                     rasa.cli.utils.print_color("Buttons:", color=color)
                     for idx, button in enumerate(buttons):
                         rasa.cli.utils.print_color(
-                            console.button_to_string(button, idx), color=color
+                            cli_utils.button_to_string(button, idx), color=color
                         )
 
             tracker = agent.tracker_store.retrieve(tracker.sender_id)
diff --git a/rasa/core/training/interactive.py b/rasa/core/training/interactive.py
index 6831dc2db90..3f8b92e4964 100644
--- a/rasa/core/training/interactive.py
+++ b/rasa/core/training/interactive.py
@@ -27,7 +27,6 @@ from rasa.core.actions.action import (
     UTTER_PREFIX,
 )
 from rasa.core.channels.channel import UserMessage
-from rasa.core.channels.channel import button_to_string, element_to_string
 from rasa.core.constants import (
     DEFAULT_SERVER_FORMAT,
     DEFAULT_SERVER_PORT,
@@ -257,20 +256,22 @@ def format_bot_output(message: BotUttered) -> Text:
 
     if data.get("buttons"):
         output += "\nButtons:"
-        for idx, button in enumerate(data.get("buttons")):
-            button_str = button_to_string(button, idx)
-            output += "\n" + button_str
+        choices = cliutils.button_choices_from_message_data(
+            data, allow_free_text_input=True
+        )
+        for choice in choices:
+            output += "\n" + choice
 
     if data.get("elements"):
         output += "\nElements:"
         for idx, element in enumerate(data.get("elements")):
-            element_str = element_to_string(element, idx)
+            element_str = cliutils.element_to_string(element, idx)
             output += "\n" + element_str
 
     if data.get("quick_replies"):
         output += "\nQuick replies:"
         for idx, element in enumerate(data.get("quick_replies")):
-            element_str = element_to_string(element, idx)
+            element_str = cliutils.element_to_string(element, idx)
             output += "\n" + element_str
     return output
 
@@ -885,7 +886,7 @@ async def _predict_till_next_listen(
     sender_ids: List[Text],
     plot_file: Optional[Text],
 ) -> None:
-    """Predict and validate actions until we need to wait for a user msg."""
+    """Predict and validate actions until we need to wait for a user message."""
 
     listen = False
     while not listen:
@@ -921,16 +922,21 @@ async def _predict_till_next_listen(
         if last_event.get("event") == BotUttered.type_name and last_event["data"].get(
             "buttons", None
         ):
-            data = last_event["data"]
-            message = last_event.get("text", "")
-            choices = [
-                button_to_string(button, idx)
-                for idx, button in enumerate(data.get("buttons"))
-            ]
+            response = _get_button_choice(last_event)
+            if response != cliutils.FREE_TEXT_INPUT_PROMPT:
+                await send_message(endpoint, sender_id, response)
+
 
-            question = questionary.select(message, choices)
-            button_payload = cliutils.payload_from_button_question(question)
-            await send_message(endpoint, sender_id, button_payload)
+def _get_button_choice(last_event: Dict[Text, Any]) -> Text:
+    data = last_event["data"]
+    message = last_event.get("text", "")
+
+    choices = cliutils.button_choices_from_message_data(
+        data, allow_free_text_input=True
+    )
+    question = questionary.select(message, choices)
+    response = cliutils.payload_from_button_question(question)
+    return response
 
 
 async def _correct_wrong_nlu(
@@ -1384,6 +1390,7 @@ async def record_messages(
                 if await is_listening_for_message(sender_id, endpoint):
                     await _enter_user_message(sender_id, endpoint)
                     await _validate_nlu(intents, endpoint, sender_id)
+
                 await _predict_till_next_listen(
                     endpoint, sender_id, sender_ids, plot_file
                 )


Test patch:
diff --git a/tests/core/test_interactive.py b/tests/core/test_interactive.py
index f831b623f0c..18599c70c1c 100644
--- a/tests/core/test_interactive.py
+++ b/tests/core/test_interactive.py
@@ -95,6 +95,7 @@ def test_bot_output_format():
         "Buttons:\n"
         "1: yes (/yes)\n"
         '2: no (/no) - {"extra": "extra"}\n'
+        "Type out your own message...\n"
         "Elements:\n"
         '1: element1 - {"buttons": '
         '[{"payload": "/button1", "title": "button1"}]'
diff --git a/tests/core/utilities.py b/tests/core/utilities.py
index 0a40805a47c..9f7038ff9e8 100644
--- a/tests/core/utilities.py
+++ b/tests/core/utilities.py
@@ -52,18 +52,18 @@ def mocked_cmd_input(package, text):
         text = [text]
 
     text_generator = itertools.cycle(text)
-    i = package.get_cmd_input
+    i = package.get_user_input
 
     def mocked_input(*args, **kwargs):
         value = next(text_generator)
         print ("wrote '{}' to input".format(value))
         return value
 
-    package.get_cmd_input = mocked_input
+    package.get_user_input = mocked_input
     try:
         yield
     finally:
-        package.get_cmd_input = i
+        package.get_user_input = i
 
 
 def user_uttered(text: Text, confidence: float) -> UserUttered: