Adding textual observations slider, decorative, changes fix for bubble newlines, mask unobserved slider only updates image

web_demo/app.py

@@ -42,7 +42,6 @@ env_label_to_env_name = {
     "Scaffolding/Formats (test)":"SocialAI-AELangFeedbackTrainFormatsCSParamEnv-v1",
 }
 
-# env = gym.make(args.env, **env_args_str_to_dict(args.env_args))
 global env_name
 global env_label
 env_label = list(env_label_to_env_name.keys())[0]
@@ -54,16 +53,23 @@ textworld_envs = ["SocialAI-AsocialBoxInformationSeekingParamEnv-v1", "SocialAI-
 global mask_unobserved
 mask_unobserved = False
 
+global textual_observations
+textual_observations = False
+
 env = gym.make(env_name)
 
+global obs, info
+obs, info = env.reset(with_info=True)
 
-def create_bubble_text(env_name, obs, info, full_conversation, textworld_envs):
-    if env_name in textworld_envs:
-        text_obs = generate_text_obs(obs, info)
-        bubble_text = text_obs
 
+def create_bubble_text(obs, info, full_conversation, textual_observations):
+    if textual_observations:
+        bubble_text = "Textual observation\n\n"+ \
+                      generate_text_obs(obs, info)
     else:
-        bubble_text = format_bubble_text(full_conversation)
+        bubble_text = full_conversation
+
+    bubble_text = format_bubble_text(bubble_text)
 
     return bubble_text
 
@@ -115,6 +121,8 @@ def set_env():
 
     global env  # Declare the env variable as global to modify it
     env = gym.make(env_name)  # Initialize the environment with the new name
+    global obs, info
+    obs, info = env.reset(with_info=True)
     update_tree()  # Update the tree for the new environment
     return redirect(url_for('index'))  # Redirect back to the main page
 
@@ -122,17 +130,30 @@ def set_env():
 @app.route('/set_mask_unobserved', methods=['POST'])
 def set_mask_unobserved():
     global mask_unobserved
-    mask_unobserved_value = request.form.get('mask_unobserved')
-    mask_unobserved = bool(mask_unobserved_value)
+    mask_unobserved = request.form.get('mask_unobserved') == 'true'
+
+    image = env.render('rgb_array', tile_size=32, mask_unobserved=mask_unobserved)
+    image_data = np_img_to_base64(image)
 
-    return redirect(url_for('index'))
+    return jsonify({'image_data': image_data})
 
+@app.route('/set_textual_observations', methods=['POST'])
+def set_textual_observations():
+    global textual_observations
+    textual_observations = request.form.get('textual_observations') == 'true'
 
+    bubble_text = create_bubble_text(obs, info, env.current_env.full_conversation, textual_observations)
 
-@app.route('/update_image', methods=['POST'])
-def update_image():
+    return jsonify({"bubble_text": bubble_text})
+
+
+
+@app.route('/perform_action', methods=['POST'])
+def perform_action():
     action_name = request.form.get('action')
 
+    global obs, info
+
     if action_name == 'done':
         # reset the env and update the tree image
         obs, info = env.reset(with_info=True)
@@ -165,7 +186,7 @@ def update_image():
     image = env.render('rgb_array', tile_size=32, mask_unobserved=mask_unobserved)
     image_data = np_img_to_base64(image)
 
-    bubble_text = create_bubble_text(env_name, obs, info, env.current_env.full_conversation, textworld_envs)
+    bubble_text = create_bubble_text(obs, info, env.current_env.full_conversation, textual_observations)
 
     return jsonify({'image_data': image_data, "bubble_text": bubble_text})
 
@@ -173,12 +194,11 @@ def update_image():
 
 @app.route('/', methods=['GET', 'POST'])
 def index():
-    obs, info = env.reset(with_info=True)
     image = env.render('rgb_array', tile_size=32, mask_unobserved=mask_unobserved)
     image_data = np_img_to_base64(image)
 
     # bubble_text = format_bubble_text(env.current_env.full_conversation)
-    bubble_text = create_bubble_text(env_name, obs, info, env.current_env.full_conversation, textworld_envs)
+    bubble_text = create_bubble_text(obs, info, env.current_env.full_conversation, textual_observations)
 
     available_env_labels = env_label_to_env_name.keys()
 

web_demo/templates/index.html

@@ -186,7 +186,7 @@
                 bodyData += `&template=${template}&word=${word}`;
             }
 
-            fetch('/update_image', {
+            fetch('/perform_action', {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/x-www-form-urlencoded',
@@ -204,7 +204,8 @@
                 // Add this to handle the caretaker's utterance
                 let bubble = document.getElementById('caretakerBubble');
                 if(data.bubble_text) {
-                    bubble.innerText = data.bubble_text;
+                    let formattedText = data.bubble_text.replace(/\n/g, '<br>');
+                    bubble.innerHTML = formattedText;
                     bubble.style.display = 'block';
                 } else {
                     bubble.style.display = 'none';
@@ -235,6 +236,56 @@
             });
         });
 
+
+        // handle the change event of the mask unobserved slider change
+        function updateMaskUnobserved() {
+            const maskUnobservedValue = document.querySelector('input[name="mask_unobserved"]').checked;
+
+            fetch('/set_mask_unobserved', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                },
+                body: `mask_unobserved=${maskUnobservedValue}`
+            })
+            .then(response => response.json())
+            .then(data => {
+                // Update the image src with the new image data
+                document.getElementById('envImage').src = `data:image/jpeg;base64,${data.image_data}`;
+            })
+            .catch(error => {
+                console.error('Error:', error);
+            });
+        }
+
+        // handle the change event of the textual observations slider change
+        function updateTextualObservations() {
+            const textualObservationsValue = document.querySelector('input[name="textual_observations"]').checked;
+
+            fetch('/set_textual_observations', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                },
+                body: `textual_observations=${textualObservationsValue}`
+            })
+            .then(response => response.json())
+            .then(data => {
+                let bubble = document.getElementById('caretakerBubble');
+                if(data.bubble_text) {
+                    let formattedText = data.bubble_text.replace(/\n/g, '<br>');
+                    bubble.innerHTML = formattedText;
+                    bubble.style.display = 'block';
+                } else {
+                    bubble.style.display = 'none';
+                }
+            })
+            .catch(error => {
+                console.error('Error:', error);
+            });
+        }
+
+
     </script>
 </head>
 <body>
@@ -252,12 +303,17 @@
 
         <div class="form-container">
             <span class="form-label">Mask unobserved cells:</span>
-            <form action="/set_mask_unobserved" method="post">
-                <label class="switch">
-                    <input type="checkbox" name="mask_unobserved" value="true" onchange="this.form.submit()" {{ 'checked' if mask_unobserved }}>
-                    <span class="slider round"></span>
-                </label>
-            </form>
+            <label class="switch">
+                <input type="checkbox" name="mask_unobserved" value="true" onchange="updateMaskUnobserved()" {% if mask_unobserved %}checked{% endif %}>
+                <span class="slider round"></span>
+            </label>
+        </div>
+        <div class="form-container">
+            <span class="form-label">Textual observation:</span>
+            <label class="switch">
+                <input type="checkbox" name="textual_observations" value="true" onchange="updateTextualObservations()" {% if textual_observations %}checked{% endif %}>
+                <span class="slider round"></span>
+            </label>
         </div>
     </div>
 
@@ -284,12 +340,12 @@
                 Speaking actions:
                 <select id="actionTemplate">
                     {% for templ in grammar_templates %}
-                    <option value="{{ templ }}" {{ 'selected' if templ == "Help" else '' }}>{{ templ }}</option>
+                        <option value="{{ templ }}" {{ 'selected' if templ == "Help" else '' }}>{{ templ }}</option>
                     {% endfor %}
                 </select>
                 <select id="actionWord">
                     {% for word in grammar_words %}
-                    <option value="{{ word }}" {{ 'selected' if word == "please" else '' }}>{{ word }}</option>
+                        <option value="{{ word }}" {{ 'selected' if word == "please" else '' }}>{{ word }}</option>
                     {% endfor %}
                 </select>
                 <button onclick="performAction('speak')">Speak [s]</button>