content

.gitattributes

@@ -32,3 +32,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+track filter=lfs diff=lfs merge=lfs -text
+content/inpainting_after.png filter=lfs diff=lfs merge=lfs -text
+content/inpainting_before.jpg filter=lfs diff=lfs merge=lfs -text
+content/inpainting_sidebar.png filter=lfs diff=lfs merge=lfs -text
+content/regen_example.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -12,7 +12,7 @@ from segmentation import segment_image
 from config import HEIGHT, WIDTH, POS_PROMPT, NEG_PROMPT, COLOR_MAPPING, map_colors, map_colors_rgb
 from palette import COLOR_MAPPING_CATEGORY
 from preprocessing import preprocess_seg_mask, get_image, get_mask
-
+from explanation import make_inpainting_explanation, make_regeneration_explanation, make_segmentation_explanation
 # wide layout
 st.set_page_config(layout="wide")
 
@@ -276,6 +276,13 @@ def main():
 
         _reset_state = check_reset_state()
 
+        if generation_mode == "Inpainting":
+            make_inpainting_explanation()
+        elif generation_mode == "Segmentation conditioning":
+            make_segmentation_explanation()
+        elif generation_mode == "Re-generate objects":
+            make_regeneration_explanation()
+
         col1, col2 = st.columns(2)
         with col1:
             make_editing_canvas(canvas_color=color_chooser,

explanation.py

@@ -0,0 +1,30 @@
+import streamlit as st
+
+def make_inpainting_explanation():
+    with st.expander("Explanation inpainting", expanded=False):
+        st.write("In the inpainting mode, you can draw regions on the input image that you want to regenerate. "
+                "This can be useful to remove unwanted objects from the image or to improve the consistency of the image."
+                )
+        st.image("content/inpainting_sidebar.png", caption="Image before inpainting, note the ornaments on the wall", width=100)
+        st.write("You can find drawing options in the sidebar. There are two modes: freedraw and polygon. Freedraw allows the user to draw with a pencil of a certain width. "
+                 "Polygon allows the user to draw a polygon by clicking on the image to add a point. The polygon is closed by right clicking.")
+
+        st.write("### Example inpainting")
+        st.write("In the example below, the ornaments on the wall are removed. The inpainting is done by drawing a mask on the image.")
+        st.image("content/inpainting_before.jpg", caption="Image before inpainting, note the ornaments on the wall", width=400)
+        st.image("content/inpainting_after.png", caption="Image before inpainting, note the ornaments on the wall", width=400)
+        
+def make_regeneration_explanation():
+    with st.expander("Explanation object regeneration"):
+        st.write("In this object regeneration mode, the model calculates which objects occur in the image. "
+                 "The user can then select which objects can be regenerated by the controlnet model by adding them in the multiselect box. "
+                 "All the object classes that are not selected will remain the same as in the original image."
+                 )
+        st.write("### Example object regeneration")
+        st.write("In the example below, the room consists of various objects such as wall, ceiling, floor, lamp, bed, ... "
+                 "In the multiselect box, all the objects except for 'lamp', 'bed and 'table' are selected to be regenerated. "
+                 )
+        st.image("content/regen_example.png", caption="Room where all concepts except for 'bed', 'lamp', 'table' are regenerated", width=400)
+
+def make_segmentation_explanation():
+    pass

test.py

@@ -1,50 +0,0 @@
-
-
-class FondantInferenceModel:
-    """FondantInferenceModel class that abstracts the model loading and inference.
-    User needs to implement an inference, pre/postprocess step and pass the class to the FondantInferenceComponent.
-    The FondantInferenceComponent will then load the model and prepare it for inference.
-    The examples folder can then show examples for a pytorch / huggingface / tensorflow / ... model.
-    """
-    def __init__(self, device: str = "cpu"):
-        self.device = device
-        # load model
-        self.model = self.load_model()
-        # set model to eval mode
-        self.eval()
-        
-    def load_model(self):
-        # load model
-        ...
-    
-    def eval(self):
-        # prepare for inference
-        self.model = self.model.eval()
-        self.model = self.model.to(self.device)
-    
-    def preprocess(self, input):
-        # preprocess input
-        ...
-    
-    def postprocess(self, output):
-        # postprocess output
-        ...
-    
-    def __call__(self, *args, **kwargs):
-        processed_inputs = self.preprocess(*args, **kwargs)
-        outputs = self.model(*processed_inputs)
-        processed_outputs = self.postprocess(outputs)
-        return processed_outputs
-    
-
-class FondantInferenceComponent(FondantTransformComponent, FondantInferenceModel):
-    # loads the model and prepares it for inference
-    
-    def transform(
-        self, args: argparse.Namespace, dataframe: dd.DataFrame
-    ) -> dd.DataFrame:
-        # by using the InferenceComponent, the model is automatically loaded and prepared for inference
-        # you just need to call the infer method
-        # the self.infer method calls the model.__call__ method of the FondantInferenceModel
-        output = self.infer(args.image)
-