Problem with inpainting Stable Diffusion and python

Dmitr: <p><strong>Inpainting Stable Diffusion</strong></p>
<p>Hi. I’m trying to change the background of a photo from a wall to a forest, but Stable Diffusion distorts the girl’s face very much. Can you look at my python code and tell me what I did wrong? I tried different models, but the result is still not very good.</p>
<pre><code class="lang-auto">import os
from dotenv import load_dotenv
import torch
import numpy as np
from diffusers import AutoPipelineForInpainting, EulerDiscreteScheduler, ControlNetModel, StableDiffusionControlNetInpaintPipeline
from PIL import Image, ImageOps
from diffusers.utils import load_image, make_image_grid
import rembg

load_dotenv()

#stable-diffusion-v1-5
SDV5_MODEL_PATH = os.getenv('SDV5_MODEL_PATH')
SAVE_PATH = os.getenv('SAVE_PATH')

#majicMIX_realistic_v6
MMIX_MODEL_PATH=os.getenv('MMIX_MODEL_PATH')

#control_v11p_sd15_openpose
CONTROLNET_MODEL_PATH=os.getenv('CONTROLNET_MODEL_PATH')

#ReV_Animated_Inpainting
REV_ANIMATED_MODEL_PATH=os.getenv('REV_ANIMATED_MODEL_PATH')

prompt = 'best quality, high definition masterpiece, trending on artstation, unrealistic volumetric fog on background, girl dancing in foggy forest'
negative_prompt = ' worst quality, low quality, normal quality, lowres, watermark, monochrome, light color, low resolution, ugly face'
num_of_img_per_prompt = 1

def binary_mask(init_image):
#init_image = load_image(url).resize((512, 768))
input_array = np.array(init_image)
mask_array = rembg.remove(input_array, only_mask=True)
mask_image = Image.fromarray(mask_array)
mask_image = ImageOps.invert(mask_image)
return mask_image

def make_inpaint_condition(init_img, mask):
init_image = np.array(init_img.convert("RGB")).astype(np.float32) / 255.0
mask_image = np.array(mask.convert("L")).astype(np.float32) / 255.0

init_image[mask_image &gt; 0.5] = -1.0  # set as masked pixel
init_image = np.expand_dims(init_image, 0).transpose(0, 3, 1, 2)
init_image = torch.from_numpy(init_image)
return init_image
if name == 'main':
controlnet = ControlNetModel.from_pretrained(CONTROLNET_MODEL_PATH, torch_dtype=torch.float32)
scheduler = EulerDiscreteScheduler.from_pretrained(REV_ANIMATED_MODEL_PATH, subfolder="scheduler")

pipeline = StableDiffusionControlNetInpaintPipeline.from_pretrained(
    MMIX_MODEL_PATH,
    controlnet=controlnet,
    torch_dtype=torch.float32
).to('cpu')

base_img = 'ddw.jpg'
init_image = load_image(base_img).resize((512, 768))

bin_mask=binary_mask(init_image)

control_image = make_inpaint_condition(init_image, bin_mask)

image = pipeline(
                 prompt=prompt,
                 scheduler=scheduler,
                 negative_prompt=negative_prompt,
                 width=512,
                 height=768,
                 num_inference_steps=150,
                 image=init_image,
                 mask_image=bin_mask,
                 control_image=control_image,
                 guidance_scale=45,
                 strength=1
                 ).images[0]

image.save('test_img.jpg')
</code></pre>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/3/1/31f1fcf366294f3565f157c4ce276c65d68f3ac0.jpeg" data-download-href="https://discuss.pytorch.org/uploads/default/31f1fcf366294f3565f157c4ce276c65d68f3ac0" title="2025-02-21_19-04-08"><img src="https://discuss.pytorch.org/uploads/default/original/3X/3/1/31f1fcf366294f3565f157c4ce276c65d68f3ac0.jpeg" alt="2025-02-21_19-04-08" data-base62-sha1="77PQTtvXN25UHZgZlVqDwIyLGec" width="333" height="500" data-dominant-color="52504A"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">2025-02-21_19-04-08</span><span class="informations">364×546 25.7 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div><br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/4/a/4a9696a4b27b61d54248d8177fbfc5b09c70cfc6.jpeg" data-download-href="https://discuss.pytorch.org/uploads/default/4a9696a4b27b61d54248d8177fbfc5b09c70cfc6" title="2025-02-21_19-03-22"><img src="https://discuss.pytorch.org/uploads/default/original/3X/4/a/4a9696a4b27b61d54248d8177fbfc5b09c70cfc6.jpeg" alt="2025-02-21_19-03-22" data-base62-sha1="aDPXXsoeD9pxa5DDZ2rFRcjlGui" width="330" height="500" data-dominant-color="637D8B"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">2025-02-21_19-03-22</span><span class="informations">363×549 42.8 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>