Upload 14 files

clip/__init__.py

@@ -0,0 +1 @@
+from .clip import *

clip/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

clip/clip.py

@@ -0,0 +1,241 @@
+import hashlib
+import os
+import urllib
+import warnings
+from typing import Any, Union, List
+
+import torch
+from PIL import Image
+from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
+from tqdm import tqdm
+
+from .model import build_model
+from .simple_tokenizer import SimpleTokenizer as _Tokenizer
+
+try:
+    from torchvision.transforms import InterpolationMode
+    BICUBIC = InterpolationMode.BICUBIC
+except ImportError:
+    BICUBIC = Image.BICUBIC
+
+
+
+__all__ = ["available_models", "load", "tokenize"]
+_tokenizer = _Tokenizer()
+
+_MODELS = {
+    "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
+    "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
+    "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
+    "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
+    "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt",
+    "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
+    "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
+    "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt",
+    "ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt",
+}
+
+
+def _download(url: str, root: str):
+    os.makedirs(root, exist_ok=True)
+    filename = os.path.basename(url)
+
+    expected_sha256 = url.split("/")[-2]
+    download_target = os.path.join(root, filename)
+
+    if os.path.exists(download_target) and not os.path.isfile(download_target):
+        raise RuntimeError(f"{download_target} exists and is not a regular file")
+
+    if os.path.isfile(download_target):
+        if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
+            return download_target
+        else:
+            warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
+
+    with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
+        with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:
+            while True:
+                buffer = source.read(8192)
+                if not buffer:
+                    break
+
+                output.write(buffer)
+                loop.update(len(buffer))
+
+    if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
+        raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match")
+
+    return download_target
+
+
+def _convert_image_to_rgb(image):
+    return image.convert("RGB")
+
+
+def _transform(n_px):
+    return Compose([
+        Resize(n_px, interpolation=BICUBIC),
+        CenterCrop(n_px),
+        _convert_image_to_rgb,
+        ToTensor(),
+        Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
+    ])
+
+
+def available_models() -> List[str]:
+    """Returns the names of available CLIP models"""
+    return list(_MODELS.keys())
+
+
+def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None):
+    """Load a CLIP model
+
+    Parameters
+    ----------
+    name : str
+        A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
+
+    device : Union[str, torch.device]
+        The device to put the loaded model
+
+    jit : bool
+        Whether to load the optimized JIT model or more hackable non-JIT model (default).
+
+    download_root: str
+        path to download the model files; by default, it uses "~/.cache/clip"
+
+    Returns
+    -------
+    model : torch.nn.Module
+        The CLIP model
+
+    preprocess : Callable[[PIL.Image], torch.Tensor]
+        A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
+    """
+    if name in _MODELS:
+        model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip"))
+    elif os.path.isfile(name):
+        model_path = name
+    else:
+        raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
+
+    with open(model_path, 'rb') as opened_file:
+        try:
+            # loading JIT archive
+            model = torch.jit.load(opened_file, map_location=device if jit else "cpu").eval()
+            state_dict = None
+        except RuntimeError:
+            # loading saved state dict
+            if jit:
+                warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
+                jit = False
+            state_dict = torch.load(opened_file, map_location="cpu")
+
+    if not jit:
+        model = build_model(state_dict or model.state_dict()).to(device)
+        if str(device) == "cpu":
+            model.float()
+        return model, _transform(model.visual.input_resolution)
+
+    # patch the device names
+    device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
+    device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
+
+    def _node_get(node: torch._C.Node, key: str):
+        """Gets attributes of a node which is polymorphic over return type.
+        
+        From https://github.com/pytorch/pytorch/pull/82628
+        """
+        sel = node.kindOf(key)
+        return getattr(node, sel)(key)
+
+    def patch_device(module):
+        try:
+            graphs = [module.graph] if hasattr(module, "graph") else []
+        except RuntimeError:
+            graphs = []
+
+        if hasattr(module, "forward1"):
+            graphs.append(module.forward1.graph)
+
+        for graph in graphs:
+            for node in graph.findAllNodes("prim::Constant"):
+                if "value" in node.attributeNames() and str(_node_get(node, "value")).startswith("cuda"):
+                    node.copyAttributes(device_node)
+
+    model.apply(patch_device)
+    patch_device(model.encode_image)
+    patch_device(model.encode_text)
+
+    # patch dtype to float32 on CPU
+    if str(device) == "cpu":
+        float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
+        float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
+        float_node = float_input.node()
+
+        def patch_float(module):
+            try:
+                graphs = [module.graph] if hasattr(module, "graph") else []
+            except RuntimeError:
+                graphs = []
+
+            if hasattr(module, "forward1"):
+                graphs.append(module.forward1.graph)
+
+            for graph in graphs:
+                for node in graph.findAllNodes("aten::to"):
+                    inputs = list(node.inputs())
+                    for i in [1, 2]:  # dtype can be the second or third argument to aten::to()
+                        if _node_get(inputs[i].node(), "value") == 5:
+                            inputs[i].node().copyAttributes(float_node)
+
+        model.apply(patch_float)
+        patch_float(model.encode_image)
+        patch_float(model.encode_text)
+
+        model.float()
+
+    return model, _transform(model.input_resolution.item())
+
+
+def tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]:
+    """
+    Returns the tokenized representation of given input string(s)
+
+    Parameters
+    ----------
+    texts : Union[str, List[str]]
+        An input string or a list of input strings to tokenize
+
+    context_length : int
+        The context length to use; all CLIP models use 77 as the context length
+
+    truncate: bool
+        Whether to truncate the text in case its encoding is longer than the context length
+
+    Returns
+    -------
+    A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length].
+    We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long.
+    """
+    if isinstance(texts, str):
+        texts = [texts]
+
+    sot_token = _tokenizer.encoder["<|startoftext|>"]
+    eot_token = _tokenizer.encoder["<|endoftext|>"]
+    all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]
+    #if packaging.version.parse(torch.__version__) < packaging.version.parse("1.8.0"):
+    #    result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+    #else:
+    result = torch.zeros(len(all_tokens), context_length, dtype=torch.int)
+
+    for i, tokens in enumerate(all_tokens):
+        if len(tokens) > context_length:
+            if truncate:
+                tokens = tokens[:context_length]
+                tokens[-1] = eot_token
+            else:
+                raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}")
+        result[i, :len(tokens)] = torch.tensor(tokens)
+
+    return result

clip/clipseg.py

@@ -0,0 +1,538 @@
+import math
+from os.path import basename, dirname, join, isfile
+import torch
+from torch import nn
+from torch.nn import functional as nnf
+from torch.nn.modules.activation import ReLU
+
+
+def get_prompt_list(prompt):
+    if prompt == 'plain':
+        return ['{}']    
+    elif prompt == 'fixed':
+        return ['a photo of a {}.']
+    elif prompt == 'shuffle':
+        return ['a photo of a {}.', 'a photograph of a {}.', 'an image of a {}.', '{}.']
+    elif prompt == 'shuffle+':
+        return ['a photo of a {}.', 'a photograph of a {}.', 'an image of a {}.', '{}.',
+                            'a cropped photo of a {}.', 'a good photo of a {}.', 'a photo of one {}.',
+                            'a bad photo of a {}.', 'a photo of the {}.']
+    else:
+        raise ValueError('Invalid value for prompt')        
+
+
+def forward_multihead_attention(x, b, with_aff=False, attn_mask=None):
+    """ 
+    Simplified version of multihead attention (taken from torch source code but without tons of if clauses). 
+    The mlp and layer norm come from CLIP.
+    x: input.
+    b: multihead attention module. 
+    """
+
+    x_ = b.ln_1(x)
+    q, k, v = nnf.linear(x_, b.attn.in_proj_weight, b.attn.in_proj_bias).chunk(3, dim=-1)
+    tgt_len, bsz, embed_dim = q.size()
+
+    head_dim = embed_dim // b.attn.num_heads
+    scaling = float(head_dim) ** -0.5
+
+    q = q.contiguous().view(tgt_len, bsz * b.attn.num_heads, b.attn.head_dim).transpose(0, 1)
+    k = k.contiguous().view(-1, bsz * b.attn.num_heads, b.attn.head_dim).transpose(0, 1)
+    v = v.contiguous().view(-1, bsz * b.attn.num_heads, b.attn.head_dim).transpose(0, 1)
+
+    q = q * scaling
+
+    attn_output_weights = torch.bmm(q, k.transpose(1, 2)) #  n_heads * batch_size, tokens^2, tokens^2
+    if attn_mask is not None:
+
+
+        attn_mask_type, attn_mask = attn_mask
+        n_heads = attn_output_weights.size(0) // attn_mask.size(0)
+        attn_mask = attn_mask.repeat(n_heads, 1)
+        
+        if attn_mask_type == 'cls_token':
+            # the mask only affects similarities compared to the readout-token.
+            attn_output_weights[:, 0, 1:] = attn_output_weights[:, 0, 1:] * attn_mask[None,...]
+            # attn_output_weights[:, 0, 0] = 0*attn_output_weights[:, 0, 0]
+
+        if attn_mask_type == 'all':
+            # print(attn_output_weights.shape, attn_mask[:, None].shape)
+            attn_output_weights[:, 1:, 1:] = attn_output_weights[:, 1:, 1:] * attn_mask[:, None]
+        
+    
+    attn_output_weights = torch.softmax(attn_output_weights, dim=-1)
+
+    attn_output = torch.bmm(attn_output_weights, v)
+    attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
+    attn_output = b.attn.out_proj(attn_output)
+
+    x = x + attn_output
+    x = x + b.mlp(b.ln_2(x))
+
+    if with_aff:
+        return x, attn_output_weights
+    else:
+        return x
+
+
+class CLIPDenseBase(nn.Module):
+
+    def __init__(self, version, reduce_cond, reduce_dim, prompt, n_tokens):
+        super().__init__()
+
+        import clip
+
+        # prec = torch.FloatTensor
+        self.clip_model, _ = clip.load(version, device='cpu', jit=False)
+        self.model = self.clip_model.visual
+
+        # if not None, scale conv weights such that we obtain n_tokens.
+        self.n_tokens = n_tokens
+
+        for p in self.clip_model.parameters():
+            p.requires_grad_(False)
+
+        # conditional
+        if reduce_cond is not None:
+            self.reduce_cond = nn.Linear(512, reduce_cond)
+            for p in self.reduce_cond.parameters():
+                p.requires_grad_(False)
+        else:
+            self.reduce_cond = None        
+
+        self.film_mul = nn.Linear(512 if reduce_cond is None else reduce_cond, reduce_dim)
+        self.film_add = nn.Linear(512 if reduce_cond is None else reduce_cond, reduce_dim)
+        
+        self.reduce = nn.Linear(768, reduce_dim)
+
+        self.prompt_list = get_prompt_list(prompt)     
+
+        # precomputed prompts
+        import pickle
+        if isfile('precomputed_prompt_vectors.pickle'):
+            precomp = pickle.load(open('precomputed_prompt_vectors.pickle', 'rb'))
+            self.precomputed_prompts = {k: torch.from_numpy(v) for k, v in precomp.items()}        
+        else:
+            self.precomputed_prompts = dict()
+    
+    def rescaled_pos_emb(self, new_size):
+        assert len(new_size) == 2
+
+        a = self.model.positional_embedding[1:].T.view(1, 768, *self.token_shape)
+        b = nnf.interpolate(a, new_size, mode='bicubic', align_corners=False).squeeze(0).view(768, new_size[0]*new_size[1]).T
+        return torch.cat([self.model.positional_embedding[:1], b])
+
+    def visual_forward(self, x_inp, extract_layers=(), skip=False, mask=None):
+        
+
+        with torch.no_grad():
+
+            inp_size = x_inp.shape[2:]
+
+            if self.n_tokens is not None:
+                stride2 = x_inp.shape[2] // self.n_tokens
+                conv_weight2 = nnf.interpolate(self.model.conv1.weight, (stride2, stride2), mode='bilinear', align_corners=True)
+                x = nnf.conv2d(x_inp, conv_weight2, bias=self.model.conv1.bias, stride=stride2, dilation=self.model.conv1.dilation)
+            else:
+                x = self.model.conv1(x_inp)  # shape = [*, width, grid, grid]
+
+            x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+            x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+
+            x = torch.cat([self.model.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+
+            standard_n_tokens = 50 if self.model.conv1.kernel_size[0] == 32 else 197
+
+            if x.shape[1] != standard_n_tokens:
+                new_shape = int(math.sqrt(x.shape[1]-1))
+                x = x + self.rescaled_pos_emb((new_shape, new_shape)).to(x.dtype)[None,:,:]
+            else:
+                x = x + self.model.positional_embedding.to(x.dtype)
+
+            x = self.model.ln_pre(x)
+
+            x = x.permute(1, 0, 2)  # NLD -> LND
+
+            activations, affinities = [], []
+            for i, res_block in enumerate(self.model.transformer.resblocks):
+                
+                if mask is not None:
+                    mask_layer, mask_type, mask_tensor = mask
+                    if mask_layer == i or mask_layer == 'all':
+                        # import ipdb; ipdb.set_trace()
+                        size = int(math.sqrt(x.shape[0] - 1))
+                        
+                        attn_mask = (mask_type, nnf.interpolate(mask_tensor.unsqueeze(1).float(), (size, size)).view(mask_tensor.shape[0], size * size))
+                        
+                    else:
+                        attn_mask = None
+                else:
+                    attn_mask = None
+
+                x, aff_per_head = forward_multihead_attention(x, res_block, with_aff=True, attn_mask=attn_mask)
+
+                if i in extract_layers:
+                    affinities += [aff_per_head]
+
+                    #if self.n_tokens is not None:
+                    #    activations += [nnf.interpolate(x, inp_size, mode='bilinear', align_corners=True)]
+                    #else:
+                    activations += [x]
+
+                if len(extract_layers) > 0 and i == max(extract_layers) and skip:
+                    print('early skip')
+                    break
+                
+            x = x.permute(1, 0, 2)  # LND -> NLD
+            x = self.model.ln_post(x[:, 0, :])
+
+            if self.model.proj is not None:
+                x = x @ self.model.proj
+
+            return x, activations, affinities
+
+    def sample_prompts(self, words, prompt_list=None):
+
+        prompt_list = prompt_list if prompt_list is not None else self.prompt_list
+
+        prompt_indices = torch.multinomial(torch.ones(len(prompt_list)), len(words), replacement=True)
+        prompts = [prompt_list[i] for i in prompt_indices]
+        return [promt.format(w) for promt, w in zip(prompts, words)]
+
+    def get_cond_vec(self, conditional, batch_size):
+        # compute conditional from a single string
+        if conditional is not None and type(conditional) == str:
+            cond = self.compute_conditional(conditional)
+            cond = cond.repeat(batch_size, 1)
+
+        # compute conditional from string list/tuple
+        elif conditional is not None and type(conditional) in {list, tuple} and type(conditional[0]) == str:
+            assert len(conditional) == batch_size
+            cond = self.compute_conditional(conditional)
+
+        # use conditional directly
+        elif conditional is not None and type(conditional) == torch.Tensor and conditional.ndim == 2:
+            cond = conditional
+
+        # compute conditional from image
+        elif conditional is not None and type(conditional) == torch.Tensor:
+            with torch.no_grad():
+                cond, _, _ = self.visual_forward(conditional)
+        else:
+            raise ValueError('invalid conditional')
+        return cond   
+
+    def compute_conditional(self, conditional):
+        import clip
+
+        dev = next(self.parameters()).device
+
+        if type(conditional) in {list, tuple}:
+            text_tokens = clip.tokenize(conditional).to(dev)
+            cond = self.clip_model.encode_text(text_tokens)
+        else:
+            if conditional in self.precomputed_prompts:
+                cond = self.precomputed_prompts[conditional].float().to(dev)
+            else:
+                text_tokens = clip.tokenize([conditional]).to(dev)
+                cond = self.clip_model.encode_text(text_tokens)[0]
+        
+        if self.shift_vector is not None:
+            return cond + self.shift_vector
+        else:
+            return cond
+
+
+def clip_load_untrained(version):
+    assert version == 'ViT-B/16'
+    from clip.model import CLIP
+    from clip.clip import _MODELS, _download
+    model = torch.jit.load(_download(_MODELS['ViT-B/16'])).eval()
+    state_dict = model.state_dict()
+
+    vision_width = state_dict["visual.conv1.weight"].shape[0]
+    vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
+    vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
+    grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
+    image_resolution = vision_patch_size * grid_size
+    embed_dim = state_dict["text_projection"].shape[1]
+    context_length = state_dict["positional_embedding"].shape[0]
+    vocab_size = state_dict["token_embedding.weight"].shape[0]
+    transformer_width = state_dict["ln_final.weight"].shape[0]
+    transformer_heads = transformer_width // 64
+    transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks")))
+
+    return CLIP(embed_dim, image_resolution, vision_layers, vision_width, vision_patch_size, 
+        context_length, vocab_size, transformer_width, transformer_heads, transformer_layers)    
+
+
+class CLIPDensePredT(CLIPDenseBase):
+
+    def __init__(self, version='ViT-B/32', extract_layers=(3, 6, 9), cond_layer=0, reduce_dim=128, n_heads=4, prompt='fixed', 
+                 extra_blocks=0, reduce_cond=None, fix_shift=False,
+                 learn_trans_conv_only=False,  limit_to_clip_only=False, upsample=False, 
+                 add_calibration=False, rev_activations=False, trans_conv=None, n_tokens=None, complex_trans_conv=False):
+        
+        super().__init__(version, reduce_cond, reduce_dim, prompt, n_tokens)
+        # device = 'cpu'
+
+        self.extract_layers = extract_layers
+        self.cond_layer = cond_layer
+        self.limit_to_clip_only = limit_to_clip_only
+        self.process_cond = None
+        self.rev_activations = rev_activations
+        
+        depth = len(extract_layers)
+
+        if add_calibration:
+            self.calibration_conds = 1
+
+        self.upsample_proj = nn.Conv2d(reduce_dim, 1, kernel_size=1) if upsample else None
+
+        self.add_activation1 = True
+
+        self.version = version
+        
+        self.token_shape = {'ViT-B/32': (7, 7), 'ViT-B/16': (14, 14)}[version]
+
+        if fix_shift:
+            # self.shift_vector = nn.Parameter(torch.load(join(dirname(basename(__file__)), 'clip_text_shift_vector.pth')), requires_grad=False)
+            self.shift_vector = nn.Parameter(torch.load(join(dirname(basename(__file__)), 'shift_text_to_vis.pth')), requires_grad=False)
+            # self.shift_vector = nn.Parameter(-1*torch.load(join(dirname(basename(__file__)), 'shift2.pth')), requires_grad=False)
+        else:
+            self.shift_vector = None
+
+        if trans_conv is None:
+            trans_conv_ks = {'ViT-B/32': (32, 32), 'ViT-B/16': (16, 16)}[version]
+        else:
+            # explicitly define transposed conv kernel size
+            trans_conv_ks = (trans_conv, trans_conv)
+
+        if not complex_trans_conv:
+            self.trans_conv = nn.ConvTranspose2d(reduce_dim, 1, trans_conv_ks, stride=trans_conv_ks)
+        else:
+            assert trans_conv_ks[0] == trans_conv_ks[1]
+
+            tp_kernels = (trans_conv_ks[0] // 4, trans_conv_ks[0] // 4)
+
+            self.trans_conv = nn.Sequential(
+                nn.Conv2d(reduce_dim, reduce_dim, kernel_size=3, padding=1),
+                nn.ReLU(),
+                nn.ConvTranspose2d(reduce_dim, reduce_dim // 2, kernel_size=tp_kernels[0], stride=tp_kernels[0]),
+                nn.ReLU(),
+                nn.ConvTranspose2d(reduce_dim // 2, 1, kernel_size=tp_kernels[1], stride=tp_kernels[1]),               
+            )
+
+#        self.trans_conv = nn.ConvTranspose2d(reduce_dim, 1, trans_conv_ks, stride=trans_conv_ks)
+        
+        assert len(self.extract_layers) == depth
+
+        self.reduces = nn.ModuleList([nn.Linear(768, reduce_dim) for _ in range(depth)])
+        self.blocks = nn.ModuleList([nn.TransformerEncoderLayer(d_model=reduce_dim, nhead=n_heads) for _ in range(len(self.extract_layers))])
+        self.extra_blocks = nn.ModuleList([nn.TransformerEncoderLayer(d_model=reduce_dim, nhead=n_heads) for _ in range(extra_blocks)])
+        
+        # refinement and trans conv
+
+        if learn_trans_conv_only:
+            for p in self.parameters():
+                p.requires_grad_(False)
+            
+            for p in self.trans_conv.parameters():
+                p.requires_grad_(True)
+
+        self.prompt_list = get_prompt_list(prompt)
+
+
+    def forward(self, inp_image, conditional=None, return_features=False, mask=None):
+
+        assert type(return_features) == bool
+
+        inp_image = inp_image.to(self.model.positional_embedding.device)
+
+        if mask is not None:
+            raise ValueError('mask not supported')
+
+        # x_inp = normalize(inp_image)
+        x_inp = inp_image
+
+        bs, dev = inp_image.shape[0], x_inp.device
+
+        cond = self.get_cond_vec(conditional, bs)
+
+        visual_q, activations, _ = self.visual_forward(x_inp, extract_layers=[0] + list(self.extract_layers))
+
+        activation1 = activations[0]
+        activations = activations[1:]
+
+        _activations = activations[::-1] if not self.rev_activations else activations
+
+        a = None
+        for i, (activation, block, reduce) in enumerate(zip(_activations, self.blocks, self.reduces)):
+            
+            if a is not None:
+                a = reduce(activation) + a
+            else:
+                a = reduce(activation)
+
+            if i == self.cond_layer:
+                if self.reduce_cond is not None:
+                    cond = self.reduce_cond(cond)
+                
+                a = self.film_mul(cond) * a + self.film_add(cond)
+
+            a = block(a)
+
+        for block in self.extra_blocks:
+            a = a + block(a)
+
+        a = a[1:].permute(1, 2, 0) # rm cls token and -> BS, Feats, Tokens
+
+        size = int(math.sqrt(a.shape[2]))
+
+        a = a.view(bs, a.shape[1], size, size)
+
+        a = self.trans_conv(a)
+
+        if self.n_tokens is not None:
+            a = nnf.interpolate(a, x_inp.shape[2:], mode='bilinear', align_corners=True) 
+
+        if self.upsample_proj is not None:
+            a = self.upsample_proj(a)
+            a = nnf.interpolate(a, x_inp.shape[2:], mode='bilinear')
+
+        if return_features:
+            return a, visual_q, cond, [activation1] + activations
+        else:
+            return a,
+
+
+
+class CLIPDensePredTMasked(CLIPDensePredT):
+
+    def __init__(self, version='ViT-B/32', extract_layers=(3, 6, 9), cond_layer=0, reduce_dim=128, n_heads=4, 
+                 prompt='fixed', extra_blocks=0, reduce_cond=None, fix_shift=False, learn_trans_conv_only=False, 
+                 refine=None, limit_to_clip_only=False, upsample=False, add_calibration=False, n_tokens=None):
+
+        super().__init__(version=version, extract_layers=extract_layers, cond_layer=cond_layer, reduce_dim=reduce_dim, 
+                         n_heads=n_heads, prompt=prompt, extra_blocks=extra_blocks, reduce_cond=reduce_cond, 
+                         fix_shift=fix_shift, learn_trans_conv_only=learn_trans_conv_only,
+                         limit_to_clip_only=limit_to_clip_only, upsample=upsample, add_calibration=add_calibration,
+                         n_tokens=n_tokens)
+
+    def visual_forward_masked(self, img_s, seg_s):
+        return super().visual_forward(img_s, mask=('all', 'cls_token', seg_s))
+
+    def forward(self, img_q, cond_or_img_s, seg_s=None, return_features=False):
+
+        if seg_s is None:
+            cond = cond_or_img_s
+        else:
+            img_s = cond_or_img_s
+
+            with torch.no_grad():
+                cond, _, _ = self.visual_forward_masked(img_s, seg_s)
+
+        return super().forward(img_q, cond, return_features=return_features)
+
+
+
+class CLIPDenseBaseline(CLIPDenseBase):
+
+    def __init__(self, version='ViT-B/32', cond_layer=0, 
+                extract_layer=9, reduce_dim=128, reduce2_dim=None, prompt='fixed', 
+                 reduce_cond=None, limit_to_clip_only=False, n_tokens=None):
+        
+        super().__init__(version, reduce_cond, reduce_dim, prompt, n_tokens)
+        device = 'cpu'
+
+        # self.cond_layer = cond_layer
+        self.extract_layer = extract_layer
+        self.limit_to_clip_only = limit_to_clip_only
+        self.shift_vector = None
+
+        self.token_shape = {'ViT-B/32': (7, 7), 'ViT-B/16': (14, 14)}[version]
+        
+        assert reduce2_dim is not None
+
+        self.reduce2 = nn.Sequential(
+            nn.Linear(reduce_dim, reduce2_dim),
+            nn.ReLU(),
+            nn.Linear(reduce2_dim, reduce_dim)
+        )
+        
+        trans_conv_ks = {'ViT-B/32': (32, 32), 'ViT-B/16': (16, 16)}[version]
+        self.trans_conv = nn.ConvTranspose2d(reduce_dim, 1, trans_conv_ks, stride=trans_conv_ks)
+
+
+    def forward(self, inp_image, conditional=None, return_features=False):
+
+        inp_image = inp_image.to(self.model.positional_embedding.device)
+
+        # x_inp = normalize(inp_image)
+        x_inp = inp_image
+
+        bs, dev = inp_image.shape[0], x_inp.device
+
+        cond = self.get_cond_vec(conditional, bs)
+
+        visual_q, activations, affinities = self.visual_forward(x_inp, extract_layers=[self.extract_layer])
+
+        a = activations[0]
+        a = self.reduce(a)
+        a = self.film_mul(cond) * a + self.film_add(cond)
+
+        if self.reduce2 is not None:
+            a = self.reduce2(a)
+
+        # the original model would execute a transformer block here
+
+        a = a[1:].permute(1, 2, 0) # rm cls token and -> BS, Feats, Tokens
+
+        size = int(math.sqrt(a.shape[2]))
+
+        a = a.view(bs, a.shape[1], size, size)
+        a = self.trans_conv(a)
+
+        if return_features:
+            return a, visual_q, cond, activations
+        else:
+            return a,
+
+
+class CLIPSegMultiLabel(nn.Module):
+
+    def __init__(self, model) -> None:
+        super().__init__()
+
+        from third_party.JoEm.data_loader import get_seen_idx, get_unseen_idx, VOC
+
+        self.pascal_classes = VOC
+
+        from clip.clipseg import CLIPDensePredT
+        from general_utils import load_model
+        # self.clipseg = load_model('rd64-vit16-neg0.2-phrasecut', strict=False)
+        self.clipseg = load_model(model, strict=False)
+        
+        self.clipseg.eval()
+
+    def forward(self, x):
+
+        bs = x.shape[0]
+        out = torch.ones(21, bs, 352, 352).to(x.device) * -10
+
+        for class_id, class_name in enumerate(self.pascal_classes):
+        
+            fac = 3 if class_name == 'background' else 1
+
+            with torch.no_grad():
+                pred = torch.sigmoid(self.clipseg(x, class_name)[0][:,0]) * fac
+
+            out[class_id] += pred
+
+
+        out = out.permute(1, 0, 2, 3)
+
+        return out
+
+        # construct output tensor
+                    

clip/model.py

@@ -0,0 +1,436 @@
+from collections import OrderedDict
+from typing import Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1):
+        super().__init__()
+
+        # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
+        self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.relu1 = nn.ReLU(inplace=True)
+
+        self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.relu2 = nn.ReLU(inplace=True)
+
+        self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
+
+        self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+        self.relu3 = nn.ReLU(inplace=True)
+
+        self.downsample = None
+        self.stride = stride
+
+        if stride > 1 or inplanes != planes * Bottleneck.expansion:
+            # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
+            self.downsample = nn.Sequential(OrderedDict([
+                ("-1", nn.AvgPool2d(stride)),
+                ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
+                ("1", nn.BatchNorm2d(planes * self.expansion))
+            ]))
+
+    def forward(self, x: torch.Tensor):
+        identity = x
+
+        out = self.relu1(self.bn1(self.conv1(x)))
+        out = self.relu2(self.bn2(self.conv2(out)))
+        out = self.avgpool(out)
+        out = self.bn3(self.conv3(out))
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu3(out)
+        return out
+
+
+class AttentionPool2d(nn.Module):
+    def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
+        super().__init__()
+        self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
+        self.num_heads = num_heads
+
+    def forward(self, x):
+        x = x.flatten(start_dim=2).permute(2, 0, 1)  # NCHW -> (HW)NC
+        x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0)  # (HW+1)NC
+        x = x + self.positional_embedding[:, None, :].to(x.dtype)  # (HW+1)NC
+        x, _ = F.multi_head_attention_forward(
+            query=x[:1], key=x, value=x,
+            embed_dim_to_check=x.shape[-1],
+            num_heads=self.num_heads,
+            q_proj_weight=self.q_proj.weight,
+            k_proj_weight=self.k_proj.weight,
+            v_proj_weight=self.v_proj.weight,
+            in_proj_weight=None,
+            in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
+            bias_k=None,
+            bias_v=None,
+            add_zero_attn=False,
+            dropout_p=0,
+            out_proj_weight=self.c_proj.weight,
+            out_proj_bias=self.c_proj.bias,
+            use_separate_proj_weight=True,
+            training=self.training,
+            need_weights=False
+        )
+        return x.squeeze(0)
+
+
+class ModifiedResNet(nn.Module):
+    """
+    A ResNet class that is similar to torchvision's but contains the following changes:
+    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
+    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
+    - The final pooling layer is a QKV attention instead of an average pool
+    """
+
+    def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
+        super().__init__()
+        self.output_dim = output_dim
+        self.input_resolution = input_resolution
+
+        # the 3-layer stem
+        self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(width // 2)
+        self.relu1 = nn.ReLU(inplace=True)
+        self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(width // 2)
+        self.relu2 = nn.ReLU(inplace=True)
+        self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(width)
+        self.relu3 = nn.ReLU(inplace=True)
+        self.avgpool = nn.AvgPool2d(2)
+
+        # residual layers
+        self._inplanes = width  # this is a *mutable* variable used during construction
+        self.layer1 = self._make_layer(width, layers[0])
+        self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
+        self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
+        self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
+
+        embed_dim = width * 32  # the ResNet feature dimension
+        self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
+
+    def _make_layer(self, planes, blocks, stride=1):
+        layers = [Bottleneck(self._inplanes, planes, stride)]
+
+        self._inplanes = planes * Bottleneck.expansion
+        for _ in range(1, blocks):
+            layers.append(Bottleneck(self._inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        def stem(x):
+            x = self.relu1(self.bn1(self.conv1(x)))
+            x = self.relu2(self.bn2(self.conv2(x)))
+            x = self.relu3(self.bn3(self.conv3(x)))
+            x = self.avgpool(x)
+            return x
+
+        x = x.type(self.conv1.weight.dtype)
+        x = stem(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.attnpool(x)
+
+        return x
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class QuickGELU(nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_model * 4)),
+            ("gelu", QuickGELU()),
+            ("c_proj", nn.Linear(d_model * 4, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x: torch.Tensor):
+        self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
+        return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
+
+    def forward(self, x: torch.Tensor):
+        x = x + self.attention(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class Transformer(nn.Module):
+    def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])
+
+    def forward(self, x: torch.Tensor):
+        return self.resblocks(x)
+
+
+class VisionTransformer(nn.Module):
+    def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
+
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads)
+
+        self.ln_post = LayerNorm(width)
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+    def forward(self, x: torch.Tensor):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        x = self.ln_post(x[:, 0, :])
+
+        if self.proj is not None:
+            x = x @ self.proj
+
+        return x
+
+
+class CLIP(nn.Module):
+    def __init__(self,
+                 embed_dim: int,
+                 # vision
+                 image_resolution: int,
+                 vision_layers: Union[Tuple[int, int, int, int], int],
+                 vision_width: int,
+                 vision_patch_size: int,
+                 # text
+                 context_length: int,
+                 vocab_size: int,
+                 transformer_width: int,
+                 transformer_heads: int,
+                 transformer_layers: int
+                 ):
+        super().__init__()
+
+        self.context_length = context_length
+
+        if isinstance(vision_layers, (tuple, list)):
+            vision_heads = vision_width * 32 // 64
+            self.visual = ModifiedResNet(
+                layers=vision_layers,
+                output_dim=embed_dim,
+                heads=vision_heads,
+                input_resolution=image_resolution,
+                width=vision_width
+            )
+        else:
+            vision_heads = vision_width // 64
+            self.visual = VisionTransformer(
+                input_resolution=image_resolution,
+                patch_size=vision_patch_size,
+                width=vision_width,
+                layers=vision_layers,
+                heads=vision_heads,
+                output_dim=embed_dim
+            )
+
+        self.transformer = Transformer(
+            width=transformer_width,
+            layers=transformer_layers,
+            heads=transformer_heads,
+            attn_mask=self.build_attention_mask()
+        )
+
+        self.vocab_size = vocab_size
+        self.token_embedding = nn.Embedding(vocab_size, transformer_width)
+        self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
+        self.ln_final = LayerNorm(transformer_width)
+
+        self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
+        self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
+
+        self.initialize_parameters()
+
+    def initialize_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+
+        if isinstance(self.visual, ModifiedResNet):
+            if self.visual.attnpool is not None:
+                std = self.visual.attnpool.c_proj.in_features ** -0.5
+                nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
+
+            for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
+                for name, param in resnet_block.named_parameters():
+                    if name.endswith("bn3.weight"):
+                        nn.init.zeros_(param)
+
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    def build_attention_mask(self):
+        # lazily create causal attention mask, with full attention between the vision tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(self.context_length, self.context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    @property
+    def dtype(self):
+        return self.visual.conv1.weight.dtype
+
+    def encode_image(self, image):
+        return self.visual(image.type(self.dtype))
+
+    def encode_text(self, text):
+        x = self.token_embedding(text).type(self.dtype)  # [batch_size, n_ctx, d_model]
+
+        x = x + self.positional_embedding.type(self.dtype)
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        x = self.ln_final(x).type(self.dtype)
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
+
+        return x
+
+    def forward(self, image, text):
+        image_features = self.encode_image(image)
+        text_features = self.encode_text(text)
+
+        # normalized features
+        image_features = image_features / image_features.norm(dim=1, keepdim=True)
+        text_features = text_features / text_features.norm(dim=1, keepdim=True)
+
+        # cosine similarity as logits
+        logit_scale = self.logit_scale.exp()
+        logits_per_image = logit_scale * image_features @ text_features.t()
+        logits_per_text = logits_per_image.t()
+
+        # shape = [global_batch_size, global_batch_size]
+        return logits_per_image, logits_per_text
+
+
+def convert_weights(model: nn.Module):
+    """Convert applicable model parameters to fp16"""
+
+    def _convert_weights_to_fp16(l):
+        if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
+            l.weight.data = l.weight.data.half()
+            if l.bias is not None:
+                l.bias.data = l.bias.data.half()
+
+        if isinstance(l, nn.MultiheadAttention):
+            for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
+                tensor = getattr(l, attr)
+                if tensor is not None:
+                    tensor.data = tensor.data.half()
+
+        for name in ["text_projection", "proj"]:
+            if hasattr(l, name):
+                attr = getattr(l, name)
+                if attr is not None:
+                    attr.data = attr.data.half()
+
+    model.apply(_convert_weights_to_fp16)
+
+
+def build_model(state_dict: dict):
+    vit = "visual.proj" in state_dict
+
+    if vit:
+        vision_width = state_dict["visual.conv1.weight"].shape[0]
+        vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
+        vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
+        grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
+        image_resolution = vision_patch_size * grid_size
+    else:
+        counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
+        vision_layers = tuple(counts)
+        vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
+        output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
+        vision_patch_size = None
+        assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
+        image_resolution = output_width * 32
+
+    embed_dim = state_dict["text_projection"].shape[1]
+    context_length = state_dict["positional_embedding"].shape[0]
+    vocab_size = state_dict["token_embedding.weight"].shape[0]
+    transformer_width = state_dict["ln_final.weight"].shape[0]
+    transformer_heads = transformer_width // 64
+    transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith("transformer.resblocks")))
+
+    model = CLIP(
+        embed_dim,
+        image_resolution, vision_layers, vision_width, vision_patch_size,
+        context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
+    )
+
+    for key in ["input_resolution", "context_length", "vocab_size"]:
+        if key in state_dict:
+            del state_dict[key]
+
+    convert_weights(model)
+    model.load_state_dict(state_dict)
+    return model.eval()

clip/simple_tokenizer.py

@@ -0,0 +1,132 @@
+import gzip
+import html
+import os
+from functools import lru_cache
+
+import ftfy
+import regex as re
+
+
+@lru_cache()
+def default_bpe():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class SimpleTokenizer(object):
+    def __init__(self, bpe_path: str = default_bpe()):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}
+        self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text

clip/vitseg.py

@@ -0,0 +1,286 @@
+import math
+from posixpath import basename, dirname, join
+# import clip
+from clip.model import convert_weights
+import torch
+import json
+from torch import nn
+from torch.nn import functional as nnf
+from torch.nn.modules import activation
+from torch.nn.modules.activation import ReLU
+from torchvision import transforms
+
+normalize = transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711))
+
+from torchvision.models import ResNet
+
+
+def process_prompts(conditional, prompt_list, conditional_map):
+    # DEPRECATED
+            
+    # randomly sample a synonym
+    words = [conditional_map[int(i)] for i in conditional]
+    words = [syns[torch.multinomial(torch.ones(len(syns)), 1, replacement=True).item()] for syns in words]
+    words = [w.replace('_', ' ') for w in words]
+
+    if prompt_list is not None:
+        prompt_indices = torch.multinomial(torch.ones(len(prompt_list)), len(words), replacement=True)
+        prompts = [prompt_list[i] for i in prompt_indices]
+    else:
+        prompts = ['a photo of {}'] * (len(words))
+
+    return [promt.format(w) for promt, w in zip(prompts, words)]
+
+
+class VITDenseBase(nn.Module):
+    
+    def rescaled_pos_emb(self, new_size):
+        assert len(new_size) == 2
+
+        a = self.model.positional_embedding[1:].T.view(1, 768, *self.token_shape)
+        b = nnf.interpolate(a, new_size, mode='bicubic', align_corners=False).squeeze(0).view(768, new_size[0]*new_size[1]).T
+        return torch.cat([self.model.positional_embedding[:1], b])
+
+    def visual_forward(self, x_inp, extract_layers=(), skip=False, mask=None):
+        
+        with torch.no_grad():
+
+            x_inp = nnf.interpolate(x_inp, (384, 384))
+
+            x = self.model.patch_embed(x_inp)
+            cls_token = self.model.cls_token.expand(x.shape[0], -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
+            if self.model.dist_token is None:
+                x = torch.cat((cls_token, x), dim=1)
+            else:
+                x = torch.cat((cls_token, self.model.dist_token.expand(x.shape[0], -1, -1), x), dim=1)
+            x = self.model.pos_drop(x + self.model.pos_embed)
+
+            activations = []
+            for i, block in enumerate(self.model.blocks):
+                x = block(x)
+
+                if i in extract_layers:
+                    # permute to be compatible with CLIP
+                    activations += [x.permute(1,0,2)]                
+
+            x = self.model.norm(x)
+            x = self.model.head(self.model.pre_logits(x[:, 0]))
+
+            # again for CLIP compatibility
+            # x = x.permute(1, 0, 2)
+
+        return x, activations, None
+
+    def sample_prompts(self, words, prompt_list=None):
+
+        prompt_list = prompt_list if prompt_list is not None else self.prompt_list
+
+        prompt_indices = torch.multinomial(torch.ones(len(prompt_list)), len(words), replacement=True)
+        prompts = [prompt_list[i] for i in prompt_indices]
+        return [promt.format(w) for promt, w in zip(prompts, words)]
+
+    def get_cond_vec(self, conditional, batch_size):
+        # compute conditional from a single string
+        if conditional is not None and type(conditional) == str:
+            cond = self.compute_conditional(conditional)
+            cond = cond.repeat(batch_size, 1)
+
+        # compute conditional from string list/tuple
+        elif conditional is not None and type(conditional) in {list, tuple} and type(conditional[0]) == str:
+            assert len(conditional) == batch_size
+            cond = self.compute_conditional(conditional)
+
+        # use conditional directly
+        elif conditional is not None and type(conditional) == torch.Tensor and conditional.ndim == 2:
+            cond = conditional
+
+        # compute conditional from image
+        elif conditional is not None and type(conditional) == torch.Tensor:
+            with torch.no_grad():
+                cond, _, _ = self.visual_forward(conditional)
+        else:
+            raise ValueError('invalid conditional')
+        return cond   
+
+    def compute_conditional(self, conditional):
+        import clip
+
+        dev = next(self.parameters()).device
+
+        if type(conditional) in {list, tuple}:
+            text_tokens = clip.tokenize(conditional).to(dev)
+            cond = self.clip_model.encode_text(text_tokens)
+        else:
+            if conditional in self.precomputed_prompts:
+                cond = self.precomputed_prompts[conditional].float().to(dev)
+            else:
+                text_tokens = clip.tokenize([conditional]).to(dev)
+                cond = self.clip_model.encode_text(text_tokens)[0]
+        
+        return cond
+
+
+class VITDensePredT(VITDenseBase):
+
+    def __init__(self, extract_layers=(3, 6, 9), cond_layer=0, reduce_dim=128, n_heads=4, prompt='fixed', 
+                 depth=3, extra_blocks=0, reduce_cond=None, fix_shift=False,
+                 learn_trans_conv_only=False, refine=None, limit_to_clip_only=False, upsample=False, 
+                 add_calibration=False, process_cond=None, not_pretrained=False):
+        super().__init__()
+        # device = 'cpu'
+
+        self.extract_layers = extract_layers
+        self.cond_layer = cond_layer
+        self.limit_to_clip_only = limit_to_clip_only
+        self.process_cond = None
+        
+        if add_calibration:
+            self.calibration_conds = 1
+
+        self.upsample_proj = nn.Conv2d(reduce_dim, 1, kernel_size=1) if upsample else None
+
+        self.add_activation1 = True
+
+        import timm 
+        self.model = timm.create_model('vit_base_patch16_384', pretrained=True)
+        self.model.head = nn.Linear(768, 512 if reduce_cond is None else reduce_cond)
+
+        for p in self.model.parameters():
+            p.requires_grad_(False)
+
+        import clip
+        self.clip_model, _ = clip.load('ViT-B/16', device='cpu', jit=False)
+        # del self.clip_model.visual
+        
+        
+        self.token_shape = (14, 14)
+
+        # conditional
+        if reduce_cond is not None:
+            self.reduce_cond = nn.Linear(512, reduce_cond)
+            for p in self.reduce_cond.parameters():
+                p.requires_grad_(False)
+        else:
+            self.reduce_cond = None
+
+        # self.film = AVAILABLE_BLOCKS['film'](512, 128)
+        self.film_mul = nn.Linear(512 if reduce_cond is None else reduce_cond, reduce_dim)
+        self.film_add = nn.Linear(512 if reduce_cond is None else reduce_cond, reduce_dim)
+        
+        # DEPRECATED
+        # self.conditional_map = {c['id']: c['synonyms'] for c in json.load(open(cond_map))}
+        
+        assert len(self.extract_layers) == depth
+
+        self.reduces = nn.ModuleList([nn.Linear(768, reduce_dim) for _ in range(depth)])
+        self.blocks = nn.ModuleList([nn.TransformerEncoderLayer(d_model=reduce_dim, nhead=n_heads) for _ in range(len(self.extract_layers))])
+        self.extra_blocks = nn.ModuleList([nn.TransformerEncoderLayer(d_model=reduce_dim, nhead=n_heads) for _ in range(extra_blocks)])
+
+        trans_conv_ks = (16, 16)
+        self.trans_conv = nn.ConvTranspose2d(reduce_dim, 1, trans_conv_ks, stride=trans_conv_ks)
+
+        # refinement and trans conv
+
+        if learn_trans_conv_only:
+            for p in self.parameters():
+                p.requires_grad_(False)
+            
+            for p in self.trans_conv.parameters():
+                p.requires_grad_(True)
+
+        if prompt == 'fixed':
+            self.prompt_list = ['a photo of a {}.']
+        elif prompt == 'shuffle':
+            self.prompt_list = ['a photo of a {}.', 'a photograph of a {}.', 'an image of a {}.', '{}.']
+        elif prompt == 'shuffle+':
+            self.prompt_list = ['a photo of a {}.', 'a photograph of a {}.', 'an image of a {}.', '{}.',
+                                'a cropped photo of a {}.', 'a good photo of a {}.', 'a photo of one {}.',
+                                'a bad photo of a {}.', 'a photo of the {}.']
+        elif prompt == 'shuffle_clip':
+            from models.clip_prompts import imagenet_templates
+            self.prompt_list = imagenet_templates
+
+        if process_cond is not None:
+            if process_cond == 'clamp' or process_cond[0] == 'clamp':
+
+                val = process_cond[1] if type(process_cond) in {list, tuple} else 0.2
+
+                def clamp_vec(x):
+                    return torch.clamp(x, -val, val)
+
+                self.process_cond = clamp_vec
+
+            elif process_cond.endswith('.pth'):
+                
+                shift = torch.load(process_cond)
+                def add_shift(x):
+                    return x + shift.to(x.device)
+
+                self.process_cond = add_shift
+
+        import pickle
+        precomp = pickle.load(open('precomputed_prompt_vectors.pickle', 'rb'))
+        self.precomputed_prompts = {k: torch.from_numpy(v) for k, v in precomp.items()}
+
+
+    def forward(self, inp_image, conditional=None, return_features=False, mask=None):
+
+        assert type(return_features) == bool
+
+        # inp_image = inp_image.to(self.model.positional_embedding.device)
+
+        if mask is not None:
+            raise ValueError('mask not supported')
+
+        # x_inp = normalize(inp_image)
+        x_inp = inp_image
+
+        bs, dev = inp_image.shape[0], x_inp.device
+
+        inp_image_size = inp_image.shape[2:]
+
+        cond = self.get_cond_vec(conditional, bs)
+
+        visual_q, activations, _ = self.visual_forward(x_inp, extract_layers=[0] + list(self.extract_layers))
+
+        activation1 = activations[0]
+        activations = activations[1:]
+
+        a = None
+        for i, (activation, block, reduce) in enumerate(zip(activations[::-1], self.blocks, self.reduces)):
+            
+            if a is not None:
+                a = reduce(activation) + a
+            else:
+                a = reduce(activation)
+
+            if i == self.cond_layer:
+                if self.reduce_cond is not None:
+                    cond = self.reduce_cond(cond)
+                
+                a = self.film_mul(cond) * a + self.film_add(cond)
+
+            a = block(a)
+
+        for block in self.extra_blocks:
+            a = a + block(a)
+
+        a = a[1:].permute(1, 2, 0) # rm cls token and -> BS, Feats, Tokens
+
+        size = int(math.sqrt(a.shape[2]))
+
+        a = a.view(bs, a.shape[1], size, size)
+
+        if self.trans_conv is not None:
+            a = self.trans_conv(a)
+
+        if self.upsample_proj is not None:
+            a = self.upsample_proj(a)
+            a = nnf.interpolate(a, x_inp.shape[2:], mode='bilinear')
+
+        a = nnf.interpolate(a, inp_image_size)
+
+        if return_features:
+            return a, visual_q, cond, [activation1] + activations
+        else:
+            return a,

ui/globals.py

@@ -0,0 +1,16 @@
+ui_restart_server = False
+
+SELECTION_FACES_DATA = None
+ui_SELECTED_INPUT_FACE_INDEX = 0
+
+ui_selected_enhancer = None
+ui_upscale = None
+ui_blend_ratio = None
+ui_input_thumbs = []
+ui_target_thumbs = []
+ui_camera_frame = None
+
+
+
+
+

ui/main.py

@@ -0,0 +1,96 @@
+import os
+import time
+import gradio as gr
+import roop.globals
+import roop.metadata
+import roop.utilities as util
+import ui.globals as uii
+
+from ui.tabs.faceswap_tab import faceswap_tab
+from ui.tabs.livecam_tab import livecam_tab
+from ui.tabs.facemgr_tab import facemgr_tab
+from ui.tabs.extras_tab import extras_tab
+from ui.tabs.settings_tab import settings_tab
+
+roop.globals.keep_fps = None
+roop.globals.keep_frames = None
+roop.globals.skip_audio = None
+roop.globals.use_batch = None
+
+
+def prepare_environment():
+    roop.globals.output_path = os.path.abspath(os.path.join(os.getcwd(), "output"))
+    os.makedirs(roop.globals.output_path, exist_ok=True)
+    if not roop.globals.CFG.use_os_temp_folder:
+        os.environ["TEMP"] = os.environ["TMP"] = os.path.abspath(os.path.join(os.getcwd(), "temp"))
+    os.makedirs(os.environ["TEMP"], exist_ok=True)
+    os.environ["GRADIO_TEMP_DIR"] = os.environ["TEMP"]
+    os.environ['GRADIO_ANALYTICS_ENABLED'] = '0'
+
+def run():
+    from roop.core import decode_execution_providers, set_display_ui
+
+    prepare_environment()
+
+    set_display_ui(show_msg)
+    if roop.globals.CFG.provider == "cuda" and util.has_cuda_device() == False:
+       roop.globals.CFG.provider = "cpu"
+
+    roop.globals.execution_providers = decode_execution_providers([roop.globals.CFG.provider])
+    gputype = util.get_device()
+    if gputype == 'cuda':
+        util.print_cuda_info()
+        
+    print(f'Using provider {roop.globals.execution_providers} - Device:{gputype}')
+    
+    run_server = True
+    uii.ui_restart_server = False
+    mycss = """
+        span {color: var(--block-info-text-color)}
+        #fixedheight {
+            max-height: 238.4px;
+            overflow-y: auto !important;
+        }
+        .image-container.svelte-1l6wqyv {height: 100%}
+
+    """
+
+    while run_server:
+        server_name = roop.globals.CFG.server_name
+        if server_name is None or len(server_name) < 1:
+            server_name = None
+        server_port = roop.globals.CFG.server_port
+        if server_port <= 0:
+            server_port = None
+        ssl_verify = False if server_name == '0.0.0.0' else True
+        with gr.Blocks(title=f'{roop.metadata.name} {roop.metadata.version}', theme=roop.globals.CFG.selected_theme, css=mycss, delete_cache=(60, 86400)) as ui:
+            with gr.Row(variant='compact'):
+                    gr.Markdown(f"### [{roop.metadata.name} {roop.metadata.version}](https://github.com/C0untFloyd/roop-unleashed)")
+                    gr.HTML(util.create_version_html(), elem_id="versions")
+            faceswap_tab()
+            livecam_tab()
+            facemgr_tab()
+            extras_tab()
+            settings_tab()
+        launch_browser = roop.globals.CFG.launch_browser
+
+        uii.ui_restart_server = False
+        try:
+            ui.queue().launch(inbrowser=launch_browser, server_name=server_name, server_port=server_port, share=roop.globals.CFG.server_share, ssl_verify=ssl_verify, prevent_thread_lock=True, show_error=True)
+        except Exception as e:
+            print(f'Exception {e} when launching Gradio Server!')
+            uii.ui_restart_server = True
+            run_server = False
+        try:
+            while uii.ui_restart_server == False:
+                time.sleep(1.0)
+
+        except (KeyboardInterrupt, OSError):
+            print("Keyboard interruption in main thread... closing server.")
+            run_server = False
+        ui.close()
+
+
+def show_msg(msg: str):
+    gr.Info(msg)
+

ui/tabs/extras_tab.py

@@ -0,0 +1,245 @@
+import os
+import gradio as gr
+import shutil
+import roop.utilities as util
+import roop.util_ffmpeg as ffmpeg
+import roop.globals
+from roop.utilities import clean_dir
+
+frame_filters_map = { 
+    "Colorize B/W Images (Deoldify Artistic)" : {"colorizer" : {"subtype": "deoldify_artistic"}},
+    "Colorize B/W Images (Deoldify Stable)" : {"colorizer" : {"subtype": "deoldify_stable"}},
+    "Background remove" : {"removebg" : {"subtype": ""}},
+    "Filter Stylize" : {"filter_generic" : {"subtype" : "stylize" }},
+    "Filter Detail Enhance" : {"filter_generic" : {"subtype" : "detailenhance" }},
+    "Filter Pencil Sketch" : {"filter_generic" : {"subtype" : "pencil" }},
+    "Filter Cartoon" : {"filter_generic" : {"subtype" : "cartoon" }},
+    "Filter C64" : {"filter_generic" : {"subtype" : "C64" }}
+    }
+
+frame_upscalers_map = {
+    "ESRGAN x2" : {"upscale" : {"subtype": "esrganx2"}},
+    "ESRGAN x4" : {"upscale" : {"subtype": "esrganx4"}},
+    "LSDIR x4" : {"upscale" : {"subtype": "lsdirx4"}}
+}
+
+def extras_tab():
+    filternames = ["None"]
+    for f in frame_filters_map.keys():
+        filternames.append(f)
+    upscalernames = ["None"]
+    for f in frame_upscalers_map.keys():
+        upscalernames.append(f)
+
+    with gr.Tab("🎉 Extras"):
+        with gr.Row():
+            files_to_process = gr.Files(label='File(s) to process', file_count="multiple", file_types=["image", "video"])
+        with gr.Row(variant='panel'):
+            with gr.Accordion(label="Video/GIF", open=False):
+                with gr.Row(variant='panel'):
+                    with gr.Column():
+                        gr.Markdown("""
+                                    # Poor man's video editor
+                                    Re-encoding uses your configuration from the Settings Tab.
+    """)
+                    with gr.Column():
+                        cut_start_time = gr.Slider(0, 1000000, value=0, label="Start Frame", step=1.0, interactive=True)
+                    with gr.Column():
+                        cut_end_time = gr.Slider(1, 1000000, value=1, label="End Frame", step=1.0, interactive=True)
+                    with gr.Column():
+                        extras_chk_encode = gr.Checkbox(label='Re-encode videos (necessary for videos with different codecs)', value=False)
+                        start_cut_video = gr.Button("Cut video")
+                        start_extract_frames = gr.Button("Extract frames")
+                        start_join_videos = gr.Button("Join videos")
+
+                with gr.Row(variant='panel'):
+                    with gr.Column():
+                        gr.Markdown("""
+                                    # Create video/gif from images
+    """)
+                    with gr.Column():
+                        extras_fps = gr.Slider(minimum=0, maximum=120, value=30, label="Video FPS", step=1.0, interactive=True)
+                        extras_images_folder = gr.Textbox(show_label=False, placeholder="/content/", interactive=True)
+                    with gr.Column():
+                        extras_chk_creategif = gr.Checkbox(label='Create GIF from video', value=False)
+                        extras_create_video=gr.Button("Create")
+                with gr.Row(variant='panel'):
+                    with gr.Column():
+                        gr.Markdown("""
+                                    # Create video from gif
+    """)
+                    with gr.Column():
+                        extras_video_fps = gr.Slider(minimum=0, maximum=120, value=0, label="Video FPS", step=1.0, interactive=True)
+                    with gr.Column():
+                        extras_create_video_from_gif=gr.Button("Create")
+                with gr.Row(variant='panel'):
+                    with gr.Column(scale=2):
+                        gr.Markdown("""
+                                    # Repair video
+
+                                    Uses FFMpeg to fix corrupt videos. 
+    """)
+                    with gr.Column():
+                        extras_repair_video=gr.Button("Repair")
+
+
+        with gr.Row(variant='panel'):
+            with gr.Accordion(label="Full frame processing", open=True):
+                with gr.Row(variant='panel'):
+                    filterselection = gr.Dropdown(filternames, value="None", label="Colorizer/FilterFX", interactive=True)
+                    upscalerselection = gr.Dropdown(upscalernames, value="None", label="Enhancer", interactive=True)
+                with gr.Row(variant='panel'):
+                    start_frame_process=gr.Button("Start processing")
+
+        with gr.Row():
+            gr.Button("👀 Open Output Folder", size='sm').click(fn=lambda: util.open_folder(roop.globals.output_path))
+        with gr.Row():
+            extra_files_output = gr.Files(label='Resulting output files', file_count="multiple")
+
+    start_cut_video.click(fn=on_cut_video, inputs=[files_to_process, cut_start_time, cut_end_time, extras_chk_encode], outputs=[extra_files_output])
+    start_extract_frames.click(fn=on_extras_extract_frames, inputs=[files_to_process], outputs=[extra_files_output])
+    start_join_videos.click(fn=on_join_videos, inputs=[files_to_process, extras_chk_encode], outputs=[extra_files_output])
+    extras_create_video.click(fn=on_extras_create_video, inputs=[files_to_process, extras_images_folder, extras_fps, extras_chk_creategif], outputs=[extra_files_output])
+    extras_create_video_from_gif.click(fn=on_extras_create_video_from_gif, inputs=[files_to_process, extras_video_fps], outputs=[extra_files_output])
+    extras_repair_video.click(fn=on_extras_repair_video, inputs=[files_to_process], outputs=[extra_files_output])
+    start_frame_process.click(fn=on_frame_process, inputs=[files_to_process, filterselection, upscalerselection], outputs=[extra_files_output])
+
+
+def on_cut_video(files, cut_start_frame, cut_end_frame, reencode):
+    if files is None:
+        return None
+    
+    resultfiles = []
+    for tf in files:
+        f = tf.name
+        destfile = util.get_destfilename_from_path(f, roop.globals.output_path, '_cut')
+        ffmpeg.cut_video(f, destfile, cut_start_frame, cut_end_frame, reencode)
+        if os.path.isfile(destfile):
+            resultfiles.append(destfile)
+        else:
+            gr.Error('Cutting video failed!')
+    return resultfiles
+
+
+def on_join_videos(files, chk_encode):
+    if files is None:
+        return None
+    
+    filenames = []
+    for f in files:
+        filenames.append(f.name)
+    destfile = util.get_destfilename_from_path(filenames[0], roop.globals.output_path, '_join')
+    sorted_filenames = util.sort_filenames_ignore_path(filenames)        
+    ffmpeg.join_videos(sorted_filenames, destfile, not chk_encode)
+    resultfiles = []
+    if os.path.isfile(destfile):
+        resultfiles.append(destfile)
+    else:
+        gr.Error('Joining videos failed!')
+    return resultfiles
+
+def on_extras_create_video_from_gif(files,fps):
+    if files is None:
+        return None
+    
+    filenames = []
+    resultfiles = []
+    for f in files:
+        filenames.append(f.name)
+
+    destfilename = os.path.join(roop.globals.output_path, "img2video." + roop.globals.CFG.output_video_format)
+    ffmpeg.create_video_from_gif(filenames[0], destfilename)
+    if os.path.isfile(destfilename):
+        resultfiles.append(destfilename)
+    return resultfiles
+
+
+def on_extras_repair_video(files):
+    if files is None:
+        return None
+    
+    resultfiles = []
+    for tf in files:
+        f = tf.name
+        destfile = util.get_destfilename_from_path(f, roop.globals.output_path, '_repair')
+        ffmpeg.repair_video(f, destfile)
+        if os.path.isfile(destfile):
+            resultfiles.append(destfile)
+        else:
+            gr.Error('Repairing video failed!')
+    return resultfiles
+
+
+
+
+
+def on_extras_create_video(files, images_path,fps, create_gif):
+    if images_path is None:
+        return None
+    resultfiles = []
+    if len(files) > 0 and util.is_video(files[0]) and create_gif:
+        destfilename = files[0]
+    else:                     
+        util.sort_rename_frames(os.path.dirname(images_path))
+        destfilename = os.path.join(roop.globals.output_path, "img2video." + roop.globals.CFG.output_video_format)
+        ffmpeg.create_video('', destfilename, fps, images_path)
+        if os.path.isfile(destfilename):
+            resultfiles.append(destfilename)
+        else:
+            return None
+    if create_gif:
+        gifname = util.get_destfilename_from_path(destfilename, './output', '.gif')
+        ffmpeg.create_gif_from_video(destfilename, gifname)
+        if os.path.isfile(destfilename):
+            resultfiles.append(gifname)
+    return resultfiles
+    
+
+def on_extras_extract_frames(files):
+    if files is None:
+        return None
+    
+    resultfiles = []
+    for tf in files:
+        f = tf.name
+        resfolder = ffmpeg.extract_frames(f)
+        for file in os.listdir(resfolder):
+            outfile = os.path.join(resfolder, file)
+            if os.path.isfile(outfile):
+                resultfiles.append(outfile)
+    return resultfiles
+
+
+def on_frame_process(files, filterselection, upscaleselection):
+    import pathlib
+    from roop.core import batch_process_with_options
+    from roop.ProcessEntry import ProcessEntry
+    from roop.ProcessOptions import ProcessOptions
+    from ui.main import prepare_environment
+
+
+    if files is None:
+        return None
+
+    if roop.globals.CFG.clear_output:
+        clean_dir(roop.globals.output_path)
+    prepare_environment()
+    list_files_process : list[ProcessEntry] = []
+
+    for tf in files:
+        list_files_process.append(ProcessEntry(tf.name, 0,0, 0))
+
+    processoroptions = {}
+    filter = next((x for x in frame_filters_map.keys() if x == filterselection), None)
+    if filter is not None:
+        processoroptions.update(frame_filters_map[filter])
+    filter = next((x for x in frame_upscalers_map.keys() if x == upscaleselection), None)
+    if filter is not None:
+        processoroptions.update(frame_upscalers_map[filter])
+    options = ProcessOptions(processoroptions, 0,  0, "all", 0, None, None, 0, 128, False, False)
+    batch_process_with_options(list_files_process, options, None)
+    outdir = pathlib.Path(roop.globals.output_path)
+    outfiles = [str(item) for item in outdir.rglob("*") if item.is_file()]
+    return outfiles
+
+

ui/tabs/facemgr_tab.py

@@ -0,0 +1,187 @@
+import os
+import shutil
+import cv2
+import gradio as gr
+import roop.utilities as util
+import roop.globals
+from roop.face_util import extract_face_images
+from roop.capturer import get_video_frame, get_video_frame_total
+from typing import List, Tuple, Optional
+from roop.typing import Frame, Face, FaceSet
+
+selected_face_index = -1
+thumbs = []
+images = []
+
+
+def facemgr_tab() -> None:
+    with gr.Tab("👨‍👩‍👧‍👦 Face Management"):
+        with gr.Row():
+            gr.Markdown("""
+                        # Create blending facesets
+                        Add multiple reference images into a faceset file.
+                        """)
+        with gr.Row():
+            videoimagefst = gr.Image(label="Cut face from video frame", height=576, interactive=False, visible=True, format="jpeg")
+        with gr.Row():
+            frame_num_fst = gr.Slider(1, 1, value=1, label="Frame Number", info='0:00:00', step=1.0, interactive=False)
+            fb_cutfromframe = gr.Button("Use faces from this frame", variant='secondary', interactive=False)
+        with gr.Row():
+            fb_facesetfile = gr.Files(label='Faceset', file_count='single', file_types=['.fsz'], interactive=True)
+            fb_files = gr.Files(label='Input Files', file_count="multiple", file_types=["image", "video"], interactive=True)
+        with gr.Row():
+            with gr.Column():
+                gr.Button("👀 Open Output Folder", size='sm').click(fn=lambda: util.open_folder(roop.globals.output_path))
+            with gr.Column():
+                gr.Markdown(' ')
+        with gr.Row():
+            faces = gr.Gallery(label="Faces in this Faceset", allow_preview=True, preview=True, height=128, object_fit="scale-down")
+        with gr.Row():
+            fb_remove = gr.Button("Remove selected", variant='secondary')
+            fb_update = gr.Button("Create/Update Faceset file", variant='primary')
+            fb_clear = gr.Button("Clear all", variant='stop')
+
+    fb_facesetfile.change(fn=on_faceset_changed, inputs=[fb_facesetfile], outputs=[faces])
+    fb_files.change(fn=on_fb_files_changed, inputs=[fb_files], outputs=[faces, videoimagefst, frame_num_fst, fb_cutfromframe])
+    fb_update.click(fn=on_update_clicked, outputs=[fb_facesetfile])
+    fb_remove.click(fn=on_remove_clicked, outputs=[faces])
+    fb_clear.click(fn=on_clear_clicked, outputs=[faces, fb_files, fb_facesetfile])
+    fb_cutfromframe.click(fn=on_cutfromframe_clicked, inputs=[fb_files, frame_num_fst], outputs=[faces])
+    frame_num_fst.release(fn=on_frame_num_fst_changed, inputs=[fb_files, frame_num_fst], outputs=[videoimagefst])
+    faces.select(fn=on_face_selected)
+
+
+def on_faceset_changed(faceset, progress=gr.Progress()) -> List[Frame]:
+    global thumbs, images
+
+    if faceset is None:
+        return thumbs
+
+    thumbs.clear()
+    filename = faceset.name
+        
+    if filename.lower().endswith('fsz'):
+        progress(0, desc="Retrieving faces from Faceset File", )      
+        unzipfolder = os.path.join(os.environ["TEMP"], 'faceset')
+        if os.path.isdir(unzipfolder):
+            shutil.rmtree(unzipfolder)
+        util.mkdir_with_umask(unzipfolder)
+        util.unzip(filename, unzipfolder)
+        for file in os.listdir(unzipfolder):
+            if file.endswith(".png"):
+                SELECTION_FACES_DATA = extract_face_images(os.path.join(unzipfolder,file),  (False, 0), 0.5)
+                if len(SELECTION_FACES_DATA) < 1:
+                    gr.Warning(f"No face detected in {file}!")
+                for f in SELECTION_FACES_DATA:
+                    image = f[1]
+                    images.append(image)
+                    thumbs.append(util.convert_to_gradio(image))
+        
+        return thumbs
+
+
+def on_fb_files_changed(inputfiles, progress=gr.Progress()) -> Tuple[List[Frame], Optional[gr.Image], Optional[gr.Slider], Optional[gr.Button]]:
+    global thumbs, images, total_frames, current_video_fps
+
+    if inputfiles is None or len(inputfiles) < 1:
+        return thumbs, None, None, None
+    
+    progress(0, desc="Retrieving faces from images", )
+    slider = None
+    video_image = None
+    cut_button = None
+    for f in inputfiles:
+        source_path = f.name
+        if util.has_image_extension(source_path):
+            slider = gr.Slider(interactive=False)
+            video_image = gr.Image(interactive=False)
+            cut_button = gr.Button(interactive=False)
+            roop.globals.source_path = source_path
+            SELECTION_FACES_DATA = extract_face_images(roop.globals.source_path,  (False, 0), 0.5)
+            for f in SELECTION_FACES_DATA:
+                image = f[1]
+                images.append(image)
+                thumbs.append(util.convert_to_gradio(image))
+        elif util.is_video(source_path) or source_path.lower().endswith('gif'):
+            total_frames = get_video_frame_total(source_path)
+            current_video_fps = util.detect_fps(source_path)
+            cut_button = gr.Button(interactive=True)
+            video_image, slider = display_video_frame(source_path, 1, total_frames)
+
+    return thumbs, video_image, slider, cut_button
+    
+
+def display_video_frame(filename: str, frame_num: int, total: int=0) -> Tuple[gr.Image, gr.Slider]:
+    global current_video_fps
+
+    current_frame = get_video_frame(filename, frame_num)
+    if current_video_fps == 0:
+        current_video_fps = 1
+    secs = (frame_num - 1) / current_video_fps
+    minutes = secs / 60
+    secs = secs % 60
+    hours = minutes / 60
+    minutes = minutes % 60
+    milliseconds = (secs - int(secs)) * 1000
+    timeinfo = f"{int(hours):0>2}:{int(minutes):0>2}:{int(secs):0>2}.{int(milliseconds):0>3}"
+    if total > 0:
+        return gr.Image(value=util.convert_to_gradio(current_frame), interactive=True), gr.Slider(info=timeinfo, minimum=1, maximum=total, interactive=True)  
+    return gr.Image(value=util.convert_to_gradio(current_frame), interactive=True), gr.Slider(info=timeinfo, interactive=True)  
+
+
+def on_face_selected(evt: gr.SelectData) -> None:
+    global selected_face_index
+
+    if evt is not None:
+        selected_face_index = evt.index
+
+def on_frame_num_fst_changed(inputfiles: List[gr.Files], frame_num: int) -> Frame:
+    filename = inputfiles[0].name
+    video_image, _ = display_video_frame(filename, frame_num, 0)
+    return video_image
+
+
+def on_cutfromframe_clicked(inputfiles: List[gr.Files], frame_num: int) -> List[Frame]:
+    global thumbs
+
+    filename = inputfiles[0].name
+    SELECTION_FACES_DATA = extract_face_images(filename,  (True, frame_num), 0.5)
+    for f in SELECTION_FACES_DATA:
+        image = f[1]
+        images.append(image)
+        thumbs.append(util.convert_to_gradio(image))
+    return thumbs
+
+
+def on_remove_clicked() -> List[Frame]:
+    global thumbs, images, selected_face_index
+
+    if len(thumbs) > selected_face_index:
+        f = thumbs.pop(selected_face_index)
+        del f
+        f = images.pop(selected_face_index)
+        del f
+    return thumbs
+
+def on_clear_clicked() -> Tuple[List[Frame], None, None]:
+    global thumbs, images
+
+    thumbs.clear()
+    images.clear()
+    return thumbs, None, None
+
+
+def on_update_clicked() -> Optional[str]:
+    if len(images) < 1:
+        gr.Warning(f"No faces to create faceset from!")
+        return None
+
+    imgnames = []
+    for index,img in enumerate(images):
+        filename = os.path.join(roop.globals.output_path, f'{index}.png')
+        cv2.imwrite(filename, img)
+        imgnames.append(filename)
+
+    finalzip = os.path.join(roop.globals.output_path, 'faceset.fsz')        
+    util.zip(imgnames, finalzip)
+    return finalzip

ui/tabs/faceswap_tab.py

@@ -0,0 +1,831 @@
+import os
+import shutil
+import pathlib
+import gradio as gr
+import roop.utilities as util
+import roop.globals
+import ui.globals
+from roop.face_util import extract_face_images, create_blank_image
+from roop.capturer import get_video_frame, get_video_frame_total, get_image_frame
+from roop.ProcessEntry import ProcessEntry
+from roop.ProcessOptions import ProcessOptions
+from roop.FaceSet import FaceSet
+from roop.utilities import clean_dir
+
+last_image = None
+
+
+IS_INPUT = True
+SELECTED_FACE_INDEX = 0
+
+SELECTED_INPUT_FACE_INDEX = 0
+SELECTED_TARGET_FACE_INDEX = 0
+
+input_faces = None
+target_faces = None
+face_selection = None
+previewimage = None
+
+selected_preview_index = 0
+
+is_processing = False            
+
+list_files_process : list[ProcessEntry] = []
+no_face_choices = ["Use untouched original frame","Retry rotated", "Skip Frame", "Skip Frame if no similar face", "Use last swapped"]
+swap_choices = ["First found", "All input faces", "All female", "All male", "All faces", "Selected face"]
+
+current_video_fps = 50
+
+manual_masking = False
+
+
+def faceswap_tab():
+    global no_face_choices, previewimage
+
+    with gr.Tab("🎭 Face Swap"):
+        with gr.Row(variant='panel'):
+            with gr.Column(scale=2):
+                with gr.Row():
+                    input_faces = gr.Gallery(label="Input faces gallery", allow_preview=False, preview=False, height=138, columns=64, object_fit="scale-down", interactive=False)
+                    target_faces = gr.Gallery(label="Target faces gallery", allow_preview=False, preview=False, height=138, columns=64, object_fit="scale-down", interactive=False)
+                with gr.Row():
+                    bt_move_left_input = gr.Button("⬅ Move left", size='sm')
+                    bt_move_right_input = gr.Button("➡ Move right", size='sm')
+                    bt_move_left_target = gr.Button("⬅ Move left", size='sm')
+                    bt_move_right_target = gr.Button("➡ Move right", size='sm')
+                with gr.Row():
+                    bt_remove_selected_input_face = gr.Button("❌ Remove selected", size='sm')
+                    bt_clear_input_faces = gr.Button("💥 Clear all", variant='stop', size='sm')
+                    bt_remove_selected_target_face = gr.Button("❌ Remove selected", size='sm')
+                    bt_add_local = gr.Button('Add local files from', size='sm')
+
+                with gr.Row():
+                    with gr.Column(scale=2):
+                        with gr.Accordion(label="Advanced Masking", open=False):
+                            chk_showmaskoffsets = gr.Checkbox(
+                                label="Show mask overlay in preview",
+                                value=False,
+                                interactive=True,
+                            )
+                            chk_restoreoriginalmouth = gr.Checkbox(
+                                label="Restore original mouth area",
+                                value=False,
+                                interactive=True,
+                            )
+                            mask_top = gr.Slider(
+                                0,
+                                1.0,
+                                value=0,
+                                label="Offset Face Top",
+                                step=0.01,
+                                interactive=True,
+                            )
+                            mask_bottom = gr.Slider(
+                                0,
+                                1.0,
+                                value=0,
+                                label="Offset Face Bottom",
+                                step=0.01,
+                                interactive=True,
+                            )
+                            mask_left = gr.Slider(
+                                0,
+                                1.0,
+                                value=0,
+                                label="Offset Face Left",
+                                step=0.01,
+                                interactive=True,
+                            )
+                            mask_right = gr.Slider(
+                                0,
+                                1.0,
+                                value=0,
+                                label="Offset Face Right",
+                                step=0.01,
+                                interactive=True,
+                            )
+                            mask_erosion = gr.Slider(
+                                1.0,
+                                3.0,
+                                value=1.0,
+                                label="Erosion Iterations",
+                                step=1.00,
+                                interactive=True,
+                            )
+                            mask_blur = gr.Slider(
+                                10.0,
+                                50.0,
+                                value=20.0,
+                                label="Blur size",
+                                step=1.00,
+                                interactive=True,
+                            )
+                            bt_toggle_masking = gr.Button(
+                                "Toggle manual masking", variant="secondary", size="sm"
+                            )
+                            selected_mask_engine = gr.Dropdown(
+                                ["None", "Clip2Seg", "DFL XSeg"],
+                                value="None",
+                                label="Face masking engine",
+                            )
+                            clip_text = gr.Textbox(
+                                label="List of objects to mask and restore back on fake face",
+                                value="cup,hands,hair,banana",
+                                interactive=False,
+                            )
+                            bt_preview_mask = gr.Button(
+                                "👥 Show Mask Preview", variant="secondary"
+                            )
+                    with gr.Column(scale=2):
+                        local_folder = gr.Textbox(show_label=False, placeholder="/content/", interactive=True)
+                with gr.Row(variant='panel'):
+                    bt_srcfiles = gr.Files(label='Source Images or Facesets', file_count="multiple", file_types=["image", ".fsz"], elem_id='filelist', height=233)
+                    bt_destfiles = gr.Files(label='Target File(s)', file_count="multiple", file_types=["image", "video"], elem_id='filelist', height=233)
+                with gr.Row(variant='panel'):
+                    gr.Markdown('')
+                    forced_fps = gr.Slider(minimum=0, maximum=120, value=0, label="Video FPS", info='Overrides detected fps if not 0', step=1.0, interactive=True, container=True)
+
+            with gr.Column(scale=2):
+                previewimage = gr.Image(label="Preview Image", height=576, interactive=False, visible=True, format=get_gradio_output_format())
+                maskimage = gr.ImageEditor(label="Manual mask Image", sources=["clipboard"], transforms="", type="numpy",
+                                             brush=gr.Brush(color_mode="fixed", colors=["rgba(255, 255, 255, 1"]), interactive=True, visible=False)
+                with gr.Row(variant='panel'):
+                    fake_preview = gr.Checkbox(label="Face swap frames", value=False)
+                    bt_refresh_preview = gr.Button("🔄 Refresh", variant='secondary', size='sm')
+                    bt_use_face_from_preview = gr.Button("Use Face from this Frame", variant='primary', size='sm')
+                with gr.Row():
+                    preview_frame_num = gr.Slider(1, 1, value=1, label="Frame Number", info='0:00:00', step=1.0, interactive=True)
+                with gr.Row():
+                    text_frame_clip = gr.Markdown('Processing frame range [0 - 0]')
+                    set_frame_start = gr.Button("⬅ Set as Start", size='sm')
+                    set_frame_end = gr.Button("➡ Set as End", size='sm')
+        with gr.Row(visible=False) as dynamic_face_selection:
+            with gr.Column(scale=2):
+                face_selection = gr.Gallery(label="Detected faces", allow_preview=False, preview=False, height=138, object_fit="cover", columns=32)
+            with gr.Column():
+                bt_faceselect = gr.Button("☑ Use selected face", size='sm')
+                bt_cancelfaceselect = gr.Button("Done", size='sm')
+            with gr.Column():
+                gr.Markdown(' ') 
+
+        with gr.Row(variant='panel'):
+            with gr.Column(scale=1):
+                selected_face_detection = gr.Dropdown(swap_choices, value="First found", label="Specify face selection for swapping")
+            with gr.Column(scale=1):
+                num_swap_steps = gr.Slider(1, 5, value=1, step=1.0, label="Number of swapping steps", info="More steps may increase likeness")
+            with gr.Column(scale=2):
+                ui.globals.ui_selected_enhancer = gr.Dropdown(["None", "Codeformer", "DMDNet", "GFPGAN", "GPEN", "Restoreformer++"], value="None", label="Select post-processing")
+
+        with gr.Row(variant='panel'):
+            with gr.Column(scale=1):
+                max_face_distance = gr.Slider(0.01, 1.0, value=0.65, label="Max Face Similarity Threshold", info="0.0 = identical 1.0 = no similarity")
+            with gr.Column(scale=1):
+                ui.globals.ui_upscale = gr.Dropdown(["128px", "256px", "512px"], value="128px", label="Subsample upscale to", interactive=True)
+            with gr.Column(scale=2):
+                ui.globals.ui_blend_ratio = gr.Slider(0.0, 1.0, value=0.65, label="Original/Enhanced image blend ratio", info="Only used with active post-processing")
+
+        with gr.Row(variant='panel'):
+            with gr.Column(scale=1):
+                video_swapping_method = gr.Dropdown(["Extract Frames to media","In-Memory processing"], value="In-Memory processing", label="Select video processing method", interactive=True)
+                no_face_action = gr.Dropdown(choices=no_face_choices, value=no_face_choices[0], label="Action on no face detected", interactive=True)
+                vr_mode = gr.Checkbox(label="VR Mode", value=False)
+            with gr.Column(scale=1):
+                with gr.Group():
+                    autorotate = gr.Checkbox(label="Auto rotate horizontal Faces", value=True)
+                    roop.globals.skip_audio = gr.Checkbox(label="Skip audio", value=False)
+                    roop.globals.keep_frames = gr.Checkbox(label="Keep Frames (relevant only when extracting frames)", value=False)
+                    roop.globals.wait_after_extraction = gr.Checkbox(label="Wait for user key press before creating video ", value=False)
+
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                bt_start = gr.Button("▶ Start", variant='primary')
+            with gr.Column():
+                bt_stop = gr.Button("⏹ Stop", variant='secondary', interactive=False)
+                gr.Button("👀 Open Output Folder", size='sm').click(fn=lambda: util.open_folder(roop.globals.output_path))
+            with gr.Column(scale=2):
+                output_method = gr.Dropdown(["File","Virtual Camera", "Both"], value="File", label="Select Output Method", interactive=True)
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                resultfiles = gr.Files(label='Processed File(s)', interactive=False)
+            with gr.Column():
+                resultimage = gr.Image(type='filepath', label='Final Image', interactive=False )
+                resultvideo = gr.Video(label='Final Video', interactive=False, visible=False)
+
+    previewinputs = [preview_frame_num, bt_destfiles, fake_preview, ui.globals.ui_selected_enhancer, selected_face_detection,
+                        max_face_distance, ui.globals.ui_blend_ratio, selected_mask_engine, clip_text, no_face_action, vr_mode, autorotate, maskimage, chk_showmaskoffsets, chk_restoreoriginalmouth, num_swap_steps, ui.globals.ui_upscale]
+    previewoutputs = [previewimage, maskimage, preview_frame_num] 
+    input_faces.select(on_select_input_face, None, None).success(fn=on_preview_frame_changed, inputs=previewinputs, outputs=previewoutputs)
+    
+    bt_move_left_input.click(fn=move_selected_input, inputs=[bt_move_left_input], outputs=[input_faces])
+    bt_move_right_input.click(fn=move_selected_input, inputs=[bt_move_right_input], outputs=[input_faces])
+    bt_move_left_target.click(fn=move_selected_target, inputs=[bt_move_left_target], outputs=[target_faces])
+    bt_move_right_target.click(fn=move_selected_target, inputs=[bt_move_right_target], outputs=[target_faces])
+
+    bt_remove_selected_input_face.click(fn=remove_selected_input_face, outputs=[input_faces])
+    bt_srcfiles.change(fn=on_srcfile_changed, show_progress='full', inputs=bt_srcfiles, outputs=[dynamic_face_selection, face_selection, input_faces, bt_srcfiles])
+
+    mask_top.release(fn=on_mask_top_changed, inputs=[mask_top], show_progress='hidden')
+    mask_bottom.release(fn=on_mask_bottom_changed, inputs=[mask_bottom], show_progress='hidden')
+    mask_left.release(fn=on_mask_left_changed, inputs=[mask_left], show_progress='hidden')
+    mask_right.release(fn=on_mask_right_changed, inputs=[mask_right], show_progress='hidden')
+    mask_erosion.release(fn=on_mask_erosion_changed, inputs=[mask_erosion], show_progress='hidden')
+    mask_blur.release(fn=on_mask_blur_changed, inputs=[mask_blur], show_progress='hidden')
+    selected_mask_engine.change(fn=on_mask_engine_changed, inputs=[selected_mask_engine], outputs=[clip_text], show_progress='hidden')
+
+    target_faces.select(on_select_target_face, None, None)
+    bt_remove_selected_target_face.click(fn=remove_selected_target_face, outputs=[target_faces])
+
+    forced_fps.change(fn=on_fps_changed, inputs=[forced_fps], show_progress='hidden')
+    bt_destfiles.change(fn=on_destfiles_changed, inputs=[bt_destfiles], outputs=[preview_frame_num, text_frame_clip], show_progress='hidden').success(fn=on_preview_frame_changed, inputs=previewinputs, outputs=previewoutputs, show_progress='hidden')
+    bt_destfiles.select(fn=on_destfiles_selected, outputs=[preview_frame_num, text_frame_clip, forced_fps], show_progress='hidden').success(fn=on_preview_frame_changed, inputs=previewinputs, outputs=previewoutputs, show_progress='hidden')
+    bt_destfiles.clear(fn=on_clear_destfiles, outputs=[target_faces, selected_face_detection])
+    resultfiles.select(fn=on_resultfiles_selected, inputs=[resultfiles], outputs=[resultimage, resultvideo])
+
+    face_selection.select(on_select_face, None, None)
+    bt_faceselect.click(fn=on_selected_face, outputs=[input_faces, target_faces, selected_face_detection])
+    bt_cancelfaceselect.click(fn=on_end_face_selection, outputs=[dynamic_face_selection, face_selection])
+
+    bt_clear_input_faces.click(fn=on_clear_input_faces, outputs=[input_faces])
+
+    bt_add_local.click(fn=on_add_local_folder, inputs=[local_folder], outputs=[bt_destfiles])
+    bt_preview_mask.click(fn=on_preview_mask, inputs=[preview_frame_num, bt_destfiles, clip_text, selected_mask_engine], outputs=[previewimage]) 
+
+    start_event = bt_start.click(fn=start_swap, 
+        inputs=[output_method, ui.globals.ui_selected_enhancer, selected_face_detection, roop.globals.keep_frames, roop.globals.wait_after_extraction,
+                    roop.globals.skip_audio, max_face_distance, ui.globals.ui_blend_ratio, selected_mask_engine, clip_text,video_swapping_method, no_face_action, vr_mode, autorotate, chk_restoreoriginalmouth, num_swap_steps, ui.globals.ui_upscale, maskimage],
+        outputs=[bt_start, bt_stop, resultfiles], show_progress='full')
+    after_swap_event = start_event.success(fn=on_resultfiles_finished, inputs=[resultfiles], outputs=[resultimage, resultvideo])
+
+    bt_stop.click(fn=stop_swap, cancels=[start_event, after_swap_event], outputs=[bt_start, bt_stop], queue=False)
+
+    bt_refresh_preview.click(fn=on_preview_frame_changed, inputs=previewinputs, outputs=previewoutputs)            
+    bt_toggle_masking.click(fn=on_toggle_masking, inputs=[previewimage, maskimage], outputs=[previewimage, maskimage])            
+    fake_preview.change(fn=on_preview_frame_changed, inputs=previewinputs, outputs=previewoutputs)
+    preview_frame_num.release(fn=on_preview_frame_changed, inputs=previewinputs, outputs=previewoutputs, show_progress='hidden', )
+    bt_use_face_from_preview.click(fn=on_use_face_from_selected, show_progress='full', inputs=[bt_destfiles, preview_frame_num], outputs=[dynamic_face_selection, face_selection, target_faces, selected_face_detection])
+    set_frame_start.click(fn=on_set_frame, inputs=[set_frame_start, preview_frame_num], outputs=[text_frame_clip])
+    set_frame_end.click(fn=on_set_frame, inputs=[set_frame_end, preview_frame_num], outputs=[text_frame_clip])
+
+
+def on_mask_top_changed(mask_offset):
+    set_mask_offset(0, mask_offset)
+
+def on_mask_bottom_changed(mask_offset):
+    set_mask_offset(1, mask_offset)
+
+def on_mask_left_changed(mask_offset):
+    set_mask_offset(2, mask_offset)
+
+def on_mask_right_changed(mask_offset):
+    set_mask_offset(3, mask_offset)
+
+def on_mask_erosion_changed(mask_offset):
+    set_mask_offset(4, mask_offset)
+def on_mask_blur_changed(mask_offset):
+    set_mask_offset(5, mask_offset)
+
+
+def set_mask_offset(index, mask_offset):
+    global SELECTED_INPUT_FACE_INDEX
+
+    if len(roop.globals.INPUT_FACESETS) > SELECTED_INPUT_FACE_INDEX:
+        offs = roop.globals.INPUT_FACESETS[SELECTED_INPUT_FACE_INDEX].faces[0].mask_offsets
+        offs[index] = mask_offset
+        if offs[0] + offs[1] > 0.99:
+            offs[0] = 0.99
+            offs[1] = 0.0
+        if offs[2] + offs[3] > 0.99:
+            offs[2] = 0.99
+            offs[3] = 0.0
+        roop.globals.INPUT_FACESETS[SELECTED_INPUT_FACE_INDEX].faces[0].mask_offsets = offs
+
+def on_mask_engine_changed(mask_engine):
+    if mask_engine == "Clip2Seg":
+        return gr.Textbox(interactive=True)
+    return gr.Textbox(interactive=False)
+
+
+def on_add_local_folder(folder):
+    files = util.get_local_files_from_folder(folder)
+    if files is None:
+        gr.Warning("Empty folder or folder not found!")
+    return files
+
+
+def on_srcfile_changed(srcfiles, progress=gr.Progress()):
+    global SELECTION_FACES_DATA, IS_INPUT, input_faces, face_selection, last_image
+    
+    IS_INPUT = True
+
+    if srcfiles is None or len(srcfiles) < 1:
+        return gr.Column(visible=False), None, ui.globals.ui_input_thumbs, None
+
+    for f in srcfiles:    
+        source_path = f.name
+        if source_path.lower().endswith('fsz'):
+            progress(0, desc="Retrieving faces from Faceset File")      
+            unzipfolder = os.path.join(os.environ["TEMP"], 'faceset')
+            if os.path.isdir(unzipfolder):
+                files = os.listdir(unzipfolder)
+                for file in files:
+                    os.remove(os.path.join(unzipfolder, file))
+            else:
+                os.makedirs(unzipfolder)
+            util.mkdir_with_umask(unzipfolder)
+            util.unzip(source_path, unzipfolder)
+            is_first = True
+            face_set = FaceSet()
+            for file in os.listdir(unzipfolder):
+                if file.endswith(".png"):
+                    filename = os.path.join(unzipfolder,file)
+                    progress(0, desc="Extracting faceset")      
+                    SELECTION_FACES_DATA = extract_face_images(filename,  (False, 0))
+                    for f in SELECTION_FACES_DATA:
+                        face = f[0]
+                        face.mask_offsets = (0,0,0,0,1,20)
+                        face_set.faces.append(face)
+                        if is_first: 
+                            image = util.convert_to_gradio(f[1])
+                            ui.globals.ui_input_thumbs.append(image)
+                            is_first = False
+                        face_set.ref_images.append(get_image_frame(filename))
+            if len(face_set.faces) > 0:
+                if len(face_set.faces) > 1:
+                    face_set.AverageEmbeddings()
+                roop.globals.INPUT_FACESETS.append(face_set)
+                                        
+        elif util.has_image_extension(source_path):
+            progress(0, desc="Retrieving faces from image")      
+            roop.globals.source_path = source_path
+            SELECTION_FACES_DATA = extract_face_images(roop.globals.source_path,  (False, 0))
+            progress(0.5, desc="Retrieving faces from image")
+            for f in SELECTION_FACES_DATA:
+                face_set = FaceSet()
+                face = f[0]
+                face.mask_offsets = (0,0,0,0,1,20)
+                face_set.faces.append(face)
+                image = util.convert_to_gradio(f[1])
+                ui.globals.ui_input_thumbs.append(image)
+                roop.globals.INPUT_FACESETS.append(face_set)
+                
+    progress(1.0)
+    return gr.Column(visible=False), None, ui.globals.ui_input_thumbs,None
+
+
+def on_select_input_face(evt: gr.SelectData):
+    global SELECTED_INPUT_FACE_INDEX
+
+    SELECTED_INPUT_FACE_INDEX = evt.index
+
+
+def remove_selected_input_face():
+    global SELECTED_INPUT_FACE_INDEX
+
+    if len(roop.globals.INPUT_FACESETS) > SELECTED_INPUT_FACE_INDEX:
+        f = roop.globals.INPUT_FACESETS.pop(SELECTED_INPUT_FACE_INDEX)
+        del f
+    if len(ui.globals.ui_input_thumbs) > SELECTED_INPUT_FACE_INDEX:
+        f = ui.globals.ui_input_thumbs.pop(SELECTED_INPUT_FACE_INDEX)
+        del f
+
+    return ui.globals.ui_input_thumbs
+
+def move_selected_input(button_text):
+    global SELECTED_INPUT_FACE_INDEX
+
+    if button_text == "⬅ Move left":
+        if SELECTED_INPUT_FACE_INDEX <= 0:
+            return ui.globals.ui_input_thumbs
+        offset = -1
+    else:
+        if len(ui.globals.ui_input_thumbs) <= SELECTED_INPUT_FACE_INDEX:
+            return ui.globals.ui_input_thumbs
+        offset = 1
+    
+    f = roop.globals.INPUT_FACESETS.pop(SELECTED_INPUT_FACE_INDEX)
+    roop.globals.INPUT_FACESETS.insert(SELECTED_INPUT_FACE_INDEX + offset, f)
+    f = ui.globals.ui_input_thumbs.pop(SELECTED_INPUT_FACE_INDEX)
+    ui.globals.ui_input_thumbs.insert(SELECTED_INPUT_FACE_INDEX + offset, f)
+    return ui.globals.ui_input_thumbs
+        
+
+def move_selected_target(button_text):
+    global SELECTED_TARGET_FACE_INDEX
+
+    if button_text == "⬅ Move left":
+        if SELECTED_TARGET_FACE_INDEX <= 0:
+            return ui.globals.ui_target_thumbs
+        offset = -1
+    else:
+        if len(ui.globals.ui_target_thumbs) <= SELECTED_TARGET_FACE_INDEX:
+            return ui.globals.ui_target_thumbs
+        offset = 1
+    
+    f = roop.globals.TARGET_FACES.pop(SELECTED_TARGET_FACE_INDEX)
+    roop.globals.TARGET_FACES.insert(SELECTED_TARGET_FACE_INDEX + offset, f)
+    f = ui.globals.ui_target_thumbs.pop(SELECTED_TARGET_FACE_INDEX)
+    ui.globals.ui_target_thumbs.insert(SELECTED_TARGET_FACE_INDEX + offset, f)
+    return ui.globals.ui_target_thumbs
+
+
+
+
+def on_select_target_face(evt: gr.SelectData):
+    global SELECTED_TARGET_FACE_INDEX
+
+    SELECTED_TARGET_FACE_INDEX = evt.index
+
+def remove_selected_target_face():
+    if len(ui.globals.ui_target_thumbs) > SELECTED_TARGET_FACE_INDEX:
+        f = roop.globals.TARGET_FACES.pop(SELECTED_TARGET_FACE_INDEX)
+        del f
+    if len(ui.globals.ui_target_thumbs) > SELECTED_TARGET_FACE_INDEX:
+        f = ui.globals.ui_target_thumbs.pop(SELECTED_TARGET_FACE_INDEX)
+        del f
+    return ui.globals.ui_target_thumbs
+
+
+def on_use_face_from_selected(files, frame_num):
+    global IS_INPUT, SELECTION_FACES_DATA
+
+    IS_INPUT = False
+    thumbs = []
+    
+    roop.globals.target_path = files[selected_preview_index].name
+    if util.is_image(roop.globals.target_path) and not roop.globals.target_path.lower().endswith(('gif')):
+        SELECTION_FACES_DATA = extract_face_images(roop.globals.target_path,  (False, 0))
+        if len(SELECTION_FACES_DATA) > 0:
+            for f in SELECTION_FACES_DATA:
+                image = util.convert_to_gradio(f[1])
+                thumbs.append(image)
+        else:
+            gr.Info('No faces detected!')
+            roop.globals.target_path = None
+                
+    elif util.is_video(roop.globals.target_path) or roop.globals.target_path.lower().endswith(('gif')):
+        selected_frame = frame_num
+        SELECTION_FACES_DATA = extract_face_images(roop.globals.target_path, (True, selected_frame))
+        if len(SELECTION_FACES_DATA) > 0:
+            for f in SELECTION_FACES_DATA:
+                image = util.convert_to_gradio(f[1])
+                thumbs.append(image)
+        else:
+            gr.Info('No faces detected!')
+            roop.globals.target_path = None
+    else:
+        gr.Info('Unknown image/video type!')
+        roop.globals.target_path = None
+
+    if len(thumbs) == 1:
+        roop.globals.TARGET_FACES.append(SELECTION_FACES_DATA[0][0])
+        ui.globals.ui_target_thumbs.append(thumbs[0])
+        return gr.Row(visible=False), None, ui.globals.ui_target_thumbs, gr.Dropdown(value='Selected face')
+
+    return gr.Row(visible=True), thumbs, gr.Gallery(visible=True), gr.Dropdown(visible=True)
+
+
+def on_select_face(evt: gr.SelectData):  # SelectData is a subclass of EventData
+    global SELECTED_FACE_INDEX
+    SELECTED_FACE_INDEX = evt.index
+
+
+def on_selected_face():
+    global IS_INPUT, SELECTED_FACE_INDEX, SELECTION_FACES_DATA
+    
+    fd = SELECTION_FACES_DATA[SELECTED_FACE_INDEX]
+    image = util.convert_to_gradio(fd[1])
+    if IS_INPUT:
+        face_set = FaceSet()
+        fd[0].mask_offsets = (0,0,0,0,1,20)
+        face_set.faces.append(fd[0])
+        roop.globals.INPUT_FACESETS.append(face_set)
+        ui.globals.ui_input_thumbs.append(image)
+        return ui.globals.ui_input_thumbs, gr.Gallery(visible=True), gr.Dropdown(visible=True)
+    else:
+        roop.globals.TARGET_FACES.append(fd[0])
+        ui.globals.ui_target_thumbs.append(image)
+        return gr.Gallery(visible=True), ui.globals.ui_target_thumbs, gr.Dropdown(value='Selected face')
+
+#        bt_faceselect.click(fn=on_selected_face, outputs=[dynamic_face_selection, face_selection, input_faces, target_faces])
+
+def on_end_face_selection():
+    return gr.Column(visible=False), None
+
+
+def on_preview_frame_changed(frame_num, files, fake_preview, enhancer, detection, face_distance, blend_ratio,
+                              selected_mask_engine, clip_text, no_face_action, vr_mode, auto_rotate, maskimage, show_face_area, restore_original_mouth, num_steps, upsample):
+    global SELECTED_INPUT_FACE_INDEX, manual_masking, current_video_fps
+
+    from roop.core import live_swap, get_processing_plugins
+
+    manual_masking = False
+    mask_offsets = (0,0,0,0)
+    if len(roop.globals.INPUT_FACESETS) > SELECTED_INPUT_FACE_INDEX:
+        if not hasattr(roop.globals.INPUT_FACESETS[SELECTED_INPUT_FACE_INDEX].faces[0], 'mask_offsets'):
+            roop.globals.INPUT_FACESETS[SELECTED_INPUT_FACE_INDEX].faces[0].mask_offsets = mask_offsets
+        mask_offsets = roop.globals.INPUT_FACESETS[SELECTED_INPUT_FACE_INDEX].faces[0].mask_offsets
+
+    timeinfo = '0:00:00'
+    if files is None or selected_preview_index >= len(files) or frame_num is None:
+        return None,None, gr.Slider(info=timeinfo)
+
+    filename = files[selected_preview_index].name
+    if util.is_video(filename) or filename.lower().endswith('gif'):
+        current_frame = get_video_frame(filename, frame_num)
+        if current_video_fps == 0:
+            current_video_fps = 1
+        secs = (frame_num - 1) / current_video_fps
+        minutes = secs / 60
+        secs = secs % 60
+        hours = minutes / 60
+        minutes = minutes % 60
+        milliseconds = (secs - int(secs)) * 1000
+        timeinfo = f"{int(hours):0>2}:{int(minutes):0>2}:{int(secs):0>2}.{int(milliseconds):0>3}"  
+    else:
+        current_frame = get_image_frame(filename)
+    if current_frame is None:
+        return None, None, gr.Slider(info=timeinfo)
+    
+    layers = None
+    if maskimage is not None:
+        layers = maskimage["layers"]
+
+    if not fake_preview or len(roop.globals.INPUT_FACESETS) < 1:
+        return gr.Image(value=util.convert_to_gradio(current_frame), visible=True), gr.ImageEditor(visible=False), gr.Slider(info=timeinfo)
+
+    roop.globals.face_swap_mode = translate_swap_mode(detection)
+    roop.globals.selected_enhancer = enhancer
+    roop.globals.distance_threshold = face_distance
+    roop.globals.blend_ratio = blend_ratio
+    roop.globals.no_face_action = index_of_no_face_action(no_face_action)
+    roop.globals.vr_mode = vr_mode
+    roop.globals.autorotate_faces = auto_rotate
+    roop.globals.subsample_size = int(upsample[:3])
+
+
+    mask_engine = map_mask_engine(selected_mask_engine, clip_text)
+
+    roop.globals.execution_threads = roop.globals.CFG.max_threads
+    mask = layers[0] if layers is not None else None
+    face_index = SELECTED_INPUT_FACE_INDEX
+    if len(roop.globals.INPUT_FACESETS) <= face_index:
+        face_index = 0
+   
+    options = ProcessOptions(get_processing_plugins(mask_engine), roop.globals.distance_threshold, roop.globals.blend_ratio,
+                              roop.globals.face_swap_mode, face_index, clip_text, maskimage, num_steps, roop.globals.subsample_size, show_face_area, restore_original_mouth)
+
+    current_frame = live_swap(current_frame, options)
+    if current_frame is None:
+        return gr.Image(visible=True), None, gr.Slider(info=timeinfo)
+    return gr.Image(value=util.convert_to_gradio(current_frame), visible=True), gr.ImageEditor(visible=False), gr.Slider(info=timeinfo)
+
+def map_mask_engine(selected_mask_engine, clip_text):
+    if selected_mask_engine == "Clip2Seg":
+        mask_engine = "mask_clip2seg"
+        if clip_text is None or len(clip_text) < 1:
+          mask_engine = None
+    elif selected_mask_engine == "DFL XSeg":
+        mask_engine = "mask_xseg"
+    else:
+        mask_engine = None
+    return mask_engine
+
+
+def on_toggle_masking(previewimage, mask):
+    global manual_masking
+
+    manual_masking = not manual_masking
+    if manual_masking:
+        layers = mask["layers"]
+        if len(layers) == 1:
+            layers = [create_blank_image(previewimage.shape[1],previewimage.shape[0])]
+        return gr.Image(visible=False), gr.ImageEditor(value={"background": previewimage, "layers": layers, "composite": None}, visible=True)
+    return gr.Image(visible=True), gr.ImageEditor(visible=False)
+
+def gen_processing_text(start, end):
+    return f'Processing frame range [{start} - {end}]'
+
+def on_set_frame(sender:str, frame_num):
+    global selected_preview_index, list_files_process
+    
+    idx = selected_preview_index
+    if list_files_process[idx].endframe == 0:
+        return gen_processing_text(0,0)
+    
+    start = list_files_process[idx].startframe
+    end = list_files_process[idx].endframe
+    if sender.lower().endswith('start'):
+        list_files_process[idx].startframe = min(frame_num, end)
+    else:
+        list_files_process[idx].endframe = max(frame_num, start)
+    
+    return gen_processing_text(list_files_process[idx].startframe,list_files_process[idx].endframe)
+
+
+def on_preview_mask(frame_num, files, clip_text, mask_engine):
+    from roop.core import live_swap, get_processing_plugins
+    global is_processing
+
+    if is_processing or files is None or selected_preview_index >= len(files) or clip_text is None or frame_num is None:
+        return None
+        
+    filename = files[selected_preview_index].name
+    if util.is_video(filename) or filename.lower().endswith('gif'):
+        current_frame = get_video_frame(filename, frame_num
+                                        )
+    else:
+        current_frame = get_image_frame(filename)
+    if current_frame is None or mask_engine is None:
+        return None
+    if mask_engine == "Clip2Seg":
+        mask_engine = "mask_clip2seg"
+        if clip_text is None or len(clip_text) < 1:
+          mask_engine = None
+    elif mask_engine == "DFL XSeg":
+        mask_engine = "mask_xseg"
+    options = ProcessOptions(get_processing_plugins(mask_engine), roop.globals.distance_threshold, roop.globals.blend_ratio,
+                              "all", 0, clip_text, None, 0, 128, False, False, True)
+
+    current_frame = live_swap(current_frame, options)
+    return util.convert_to_gradio(current_frame)
+
+
+def on_clear_input_faces():
+    ui.globals.ui_input_thumbs.clear()
+    roop.globals.INPUT_FACESETS.clear()
+    return ui.globals.ui_input_thumbs
+
+def on_clear_destfiles():
+    roop.globals.TARGET_FACES.clear()
+    ui.globals.ui_target_thumbs.clear()
+    return ui.globals.ui_target_thumbs, gr.Dropdown(value="First found")    
+
+
+def index_of_no_face_action(dropdown_text):
+    global no_face_choices
+
+    return no_face_choices.index(dropdown_text) 
+
+def translate_swap_mode(dropdown_text):
+    if dropdown_text == "Selected face":
+        return "selected"
+    elif dropdown_text == "First found":
+        return "first"
+    elif dropdown_text == "All input faces":
+        return "all_input"
+    elif dropdown_text == "All female":
+        return "all_female"
+    elif dropdown_text == "All male":
+        return "all_male"
+    
+    return "all"
+
+
+def start_swap( output_method, enhancer, detection, keep_frames, wait_after_extraction, skip_audio, face_distance, blend_ratio,
+                selected_mask_engine, clip_text, processing_method, no_face_action, vr_mode, autorotate, restore_original_mouth, num_swap_steps, upsample, imagemask, progress=gr.Progress()):
+    from ui.main import prepare_environment
+    from roop.core import batch_process_regular
+    global is_processing, list_files_process
+
+    if list_files_process is None or len(list_files_process) <= 0:
+        return gr.Button(variant="primary"), None, None
+    
+    if roop.globals.CFG.clear_output:
+        clean_dir(roop.globals.output_path)
+
+    if not util.is_installed("ffmpeg"):
+        msg = "ffmpeg is not installed! No video processing possible."
+        gr.Warning(msg)
+
+    prepare_environment()
+
+    roop.globals.selected_enhancer = enhancer
+    roop.globals.target_path = None
+    roop.globals.distance_threshold = face_distance
+    roop.globals.blend_ratio = blend_ratio
+    roop.globals.keep_frames = keep_frames
+    roop.globals.wait_after_extraction = wait_after_extraction
+    roop.globals.skip_audio = skip_audio
+    roop.globals.face_swap_mode = translate_swap_mode(detection)
+    roop.globals.no_face_action = index_of_no_face_action(no_face_action)
+    roop.globals.vr_mode = vr_mode
+    roop.globals.autorotate_faces = autorotate
+    roop.globals.subsample_size = int(upsample[:3])
+    mask_engine = map_mask_engine(selected_mask_engine, clip_text)
+
+    if roop.globals.face_swap_mode == 'selected':
+        if len(roop.globals.TARGET_FACES) < 1:
+            gr.Error('No Target Face selected!')
+            return gr.Button(variant="primary"), None, None
+
+    is_processing = True            
+    yield gr.Button(variant="secondary", interactive=False), gr.Button(variant="primary", interactive=True), None
+    roop.globals.execution_threads = roop.globals.CFG.max_threads
+    roop.globals.video_encoder = roop.globals.CFG.output_video_codec
+    roop.globals.video_quality = roop.globals.CFG.video_quality
+    roop.globals.max_memory = roop.globals.CFG.memory_limit if roop.globals.CFG.memory_limit > 0 else None
+
+    batch_process_regular(output_method, list_files_process, mask_engine, clip_text, processing_method == "In-Memory processing", imagemask, restore_original_mouth, num_swap_steps, progress, SELECTED_INPUT_FACE_INDEX)
+    is_processing = False
+    outdir = pathlib.Path(roop.globals.output_path)
+    outfiles = [str(item) for item in outdir.rglob("*") if item.is_file()]
+    if len(outfiles) > 0:
+        yield gr.Button(variant="primary", interactive=True),gr.Button(variant="secondary", interactive=False),gr.Files(value=outfiles)
+    else:
+        yield gr.Button(variant="primary", interactive=True),gr.Button(variant="secondary", interactive=False),None
+
+
+def stop_swap():
+    roop.globals.processing = False
+    gr.Info('Aborting processing - please wait for the remaining threads to be stopped')
+    return gr.Button(variant="primary", interactive=True),gr.Button(variant="secondary", interactive=False),None
+
+
+def on_fps_changed(fps):
+    global selected_preview_index, list_files_process
+
+    if len(list_files_process) < 1 or list_files_process[selected_preview_index].endframe < 1:
+        return
+    list_files_process[selected_preview_index].fps = fps
+
+
+def on_destfiles_changed(destfiles):
+    global selected_preview_index, list_files_process, current_video_fps
+
+    if destfiles is None or len(destfiles) < 1:
+        list_files_process.clear()
+        return gr.Slider(value=1, maximum=1, info='0:00:00'), ''
+    
+    for f in destfiles:
+        list_files_process.append(ProcessEntry(f.name, 0,0, 0))
+
+    selected_preview_index = 0
+    idx = selected_preview_index    
+    
+    filename = list_files_process[idx].filename
+    
+    if util.is_video(filename) or filename.lower().endswith('gif'):
+        total_frames = get_video_frame_total(filename)
+        if total_frames is None or total_frames < 1:
+            total_frames = 1
+            gr.Warning(f"Corrupted video {filename}, can't detect number of frames!")
+        else:
+            current_video_fps = util.detect_fps(filename)
+    else:
+        total_frames = 1
+    list_files_process[idx].endframe = total_frames
+    if total_frames > 1:
+        return gr.Slider(value=1, maximum=total_frames, info='0:00:00'), gen_processing_text(list_files_process[idx].startframe,list_files_process[idx].endframe)
+    return gr.Slider(value=1, maximum=total_frames, info='0:00:00'), ''
+
+
+def on_destfiles_selected(evt: gr.SelectData):
+    global selected_preview_index, list_files_process, current_video_fps
+
+    if evt is not None:
+        selected_preview_index = evt.index
+    idx = selected_preview_index    
+    filename = list_files_process[idx].filename
+    fps = list_files_process[idx].fps
+    if util.is_video(filename) or filename.lower().endswith('gif'):
+        total_frames = get_video_frame_total(filename)
+        current_video_fps = util.detect_fps(filename)
+        if list_files_process[idx].endframe == 0:
+            list_files_process[idx].endframe = total_frames 
+    else:
+        total_frames = 1
+    
+    if total_frames > 1:
+        return gr.Slider(value=list_files_process[idx].startframe, maximum=total_frames, info='0:00:00'), gen_processing_text(list_files_process[idx].startframe,list_files_process[idx].endframe), fps
+    return gr.Slider(value=1, maximum=total_frames, info='0:00:00'), gen_processing_text(0,0), fps
+
+
+def on_resultfiles_selected(evt: gr.SelectData, files):
+    selected_index = evt.index
+    filename = files[selected_index].name
+    return display_output(filename)
+
+def on_resultfiles_finished(files):
+    selected_index = 0
+    if files is None or len(files) < 1:
+        return None, None
+    
+    filename = files[selected_index].name
+    return display_output(filename)
+
+
+def get_gradio_output_format():
+    if roop.globals.CFG.output_image_format == "jpg":
+        return "jpeg"
+    return roop.globals.CFG.output_image_format
+
+
+def display_output(filename):
+    if util.is_video(filename) and roop.globals.CFG.output_show_video:
+        return gr.Image(visible=False), gr.Video(visible=True, value=filename)
+    else:
+        if util.is_video(filename) or filename.lower().endswith('gif'):
+            current_frame = get_video_frame(filename)
+        else:
+            current_frame = get_image_frame(filename)
+        return gr.Image(visible=True, value=util.convert_to_gradio(current_frame)), gr.Video(visible=False)

ui/tabs/livecam_tab.py

@@ -0,0 +1,57 @@
+import gradio as gr
+import roop.globals
+import ui.globals
+
+
+camera_frame = None
+
+def livecam_tab():
+    with gr.Tab("🎥 Live Cam"):
+        with gr.Row(variant='panel'):
+            gr.Markdown("""
+                        This feature will allow you to use your physical webcam and apply the selected faces to the stream. 
+                        You can also forward the stream to a virtual camera, which can be used in video calls or streaming software.<br />
+                        Supported are: v4l2loopback (linux), OBS Virtual Camera (macOS/Windows) and unitycapture (Windows).<br />
+                        **Please note:** to change the face or any other settings you need to stop and restart a running live cam.
+            """)
+
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                bt_start = gr.Button("▶ Start", variant='primary')
+            with gr.Column():
+                bt_stop = gr.Button("⏹ Stop", variant='secondary', interactive=False)
+            with gr.Column():
+                camera_num = gr.Slider(0, 8, value=0, label="Camera Number", step=1.0, interactive=True)
+                cb_obs = gr.Checkbox(label="Forward stream to virtual camera", interactive=True)
+            with gr.Column():
+                dd_reso = gr.Dropdown(choices=["640x480","1280x720", "1920x1080"], value="1280x720", label="Fake Camera Resolution", interactive=True)
+                cb_xseg = gr.Checkbox(label="Use DFL Xseg masking", interactive=True, value=True)
+                cb_mouthrestore = gr.Checkbox(label="Restore original mouth area", interactive=True, value=False)
+
+        with gr.Row():
+            fake_cam_image = gr.Image(label='Fake Camera Output', interactive=False, format="jpeg")
+
+    start_event = bt_start.click(fn=start_cam,  inputs=[cb_obs, cb_xseg, cb_mouthrestore, camera_num, dd_reso, ui.globals.ui_selected_enhancer, ui.globals.ui_blend_ratio, ui.globals.ui_upscale],outputs=[bt_start, bt_stop,fake_cam_image])
+    bt_stop.click(fn=stop_swap, cancels=[start_event], outputs=[bt_start, bt_stop], queue=False)
+
+
+def start_cam(stream_to_obs, use_xseg, use_mouthrestore, cam, reso, enhancer, blend_ratio, upscale):
+    from roop.virtualcam import start_virtual_cam
+    from roop.utilities import convert_to_gradio
+
+    roop.globals.selected_enhancer = enhancer
+    roop.globals.blend_ratio = blend_ratio
+    roop.globals.subsample_size = int(upscale[:3])
+    start_virtual_cam(stream_to_obs, use_xseg, use_mouthrestore, cam, reso)
+    while True:
+        yield gr.Button(interactive=False), gr.Button(interactive=True), convert_to_gradio(ui.globals.ui_camera_frame)
+        
+
+def stop_swap():
+    from roop.virtualcam import stop_virtual_cam
+    stop_virtual_cam()
+    return gr.Button(interactive=True), gr.Button(interactive=False)
+    
+
+
+

ui/tabs/settings_tab.py

@@ -0,0 +1,129 @@
+import shutil
+import os
+import gradio as gr
+import roop.globals
+import ui.globals
+from roop.utilities import clean_dir
+
+available_themes = ["Default", "gradio/glass", "gradio/monochrome", "gradio/seafoam", "gradio/soft", "gstaff/xkcd", "freddyaboulton/dracula_revamped", "ysharma/steampunk"]
+image_formats = ['jpg','png', 'webp']
+video_formats = ['avi','mkv', 'mp4', 'webm']
+video_codecs = ['libx264', 'libx265', 'libvpx-vp9', 'h264_nvenc', 'hevc_nvenc']
+providerlist = None
+
+settings_controls = []
+
+def settings_tab():
+    from roop.core import suggest_execution_providers
+    global providerlist
+
+    providerlist = suggest_execution_providers()
+    with gr.Tab("⚙ Settings"):
+        with gr.Row():
+            with gr.Column():
+                themes = gr.Dropdown(available_themes, label="Theme", info="Change needs complete restart", value=roop.globals.CFG.selected_theme)
+            with gr.Column():
+                settings_controls.append(gr.Checkbox(label="Public Server", value=roop.globals.CFG.server_share, elem_id='server_share', interactive=True))
+                settings_controls.append(gr.Checkbox(label='Clear output folder before each run', value=roop.globals.CFG.clear_output, elem_id='clear_output', interactive=True))
+                output_template = gr.Textbox(label="Filename Output Template", info="(file extension is added automatically)", lines=1, placeholder='{file}_{time}', value=roop.globals.CFG.output_template)
+            with gr.Column():
+                input_server_name = gr.Textbox(label="Server Name", lines=1, info="Leave blank to run locally", value=roop.globals.CFG.server_name)
+            with gr.Column():
+                input_server_port = gr.Number(label="Server Port", precision=0, info="Leave at 0 to use default", value=roop.globals.CFG.server_port)
+        with gr.Row():
+            with gr.Column():
+                settings_controls.append(gr.Dropdown(providerlist, label="Provider", value=roop.globals.CFG.provider, elem_id='provider', interactive=True))
+                chk_det_size = gr.Checkbox(label="Use default Det-Size", value=True, elem_id='default_det_size', interactive=True)
+                settings_controls.append(gr.Checkbox(label="Force CPU for Face Analyser", value=roop.globals.CFG.force_cpu, elem_id='force_cpu', interactive=True))
+                max_threads = gr.Slider(1, 32, value=roop.globals.CFG.max_threads, label="Max. Number of Threads", info='default: 3', step=1.0, interactive=True)
+            with gr.Column():
+                memory_limit = gr.Slider(0, 128, value=roop.globals.CFG.memory_limit, label="Max. Memory to use (Gb)", info='0 meaning no limit', step=1.0, interactive=True)
+                settings_controls.append(gr.Dropdown(image_formats, label="Image Output Format", info='default: png', value=roop.globals.CFG.output_image_format, elem_id='output_image_format', interactive=True))
+            with gr.Column():
+                settings_controls.append(gr.Dropdown(video_codecs, label="Video Codec", info='default: libx264', value=roop.globals.CFG.output_video_codec, elem_id='output_video_codec', interactive=True))
+                settings_controls.append(gr.Dropdown(video_formats, label="Video Output Format", info='default: mp4', value=roop.globals.CFG.output_video_format, elem_id='output_video_format', interactive=True))
+                video_quality = gr.Slider(0, 100, value=roop.globals.CFG.video_quality, label="Video Quality (crf)", info='default: 14', step=1.0, interactive=True)
+            with gr.Column():
+                with gr.Group():
+                    settings_controls.append(gr.Checkbox(label='Use OS temp folder', value=roop.globals.CFG.use_os_temp_folder, elem_id='use_os_temp_folder', interactive=True))
+                    settings_controls.append(gr.Checkbox(label='Show video in browser (re-encodes output)', value=roop.globals.CFG.output_show_video, elem_id='output_show_video', interactive=True))
+                button_apply_restart = gr.Button("Restart Server", variant='primary')
+                button_clean_temp = gr.Button("Clean temp folder")
+                button_apply_settings = gr.Button("Apply Settings")
+
+    chk_det_size.select(fn=on_option_changed)
+
+    # Settings
+    for s in settings_controls:
+        s.select(fn=on_settings_changed)
+    max_threads.input(fn=lambda a,b='max_threads':on_settings_changed_misc(a,b), inputs=[max_threads])
+    memory_limit.input(fn=lambda a,b='memory_limit':on_settings_changed_misc(a,b), inputs=[memory_limit])
+    video_quality.input(fn=lambda a,b='video_quality':on_settings_changed_misc(a,b), inputs=[video_quality])
+
+    # button_clean_temp.click(fn=clean_temp, outputs=[bt_srcfiles, input_faces, target_faces, bt_destfiles])
+    button_clean_temp.click(fn=clean_temp)
+    button_apply_settings.click(apply_settings, inputs=[themes, input_server_name, input_server_port, output_template])
+    button_apply_restart.click(restart)
+
+
+def on_option_changed(evt: gr.SelectData):
+    attribname = evt.target.elem_id
+    if isinstance(evt.target, gr.Checkbox):
+        if hasattr(roop.globals, attribname):
+            setattr(roop.globals, attribname, evt.selected)
+            return
+    elif isinstance(evt.target, gr.Dropdown):
+        if hasattr(roop.globals, attribname):
+            setattr(roop.globals, attribname, evt.value)
+            return
+    raise gr.Error(f'Unhandled Setting for {evt.target}')
+
+
+def on_settings_changed_misc(new_val, attribname):
+    if hasattr(roop.globals.CFG, attribname):
+        setattr(roop.globals.CFG, attribname, new_val)
+    else:
+        print("Didn't find attrib!")
+        
+
+
+def on_settings_changed(evt: gr.SelectData):
+    attribname = evt.target.elem_id
+    if isinstance(evt.target, gr.Checkbox):
+        if hasattr(roop.globals.CFG, attribname):
+            setattr(roop.globals.CFG, attribname, evt.selected)
+            return
+    elif isinstance(evt.target, gr.Dropdown):
+        if hasattr(roop.globals.CFG, attribname):
+            setattr(roop.globals.CFG, attribname, evt.value)
+            return
+            
+    raise gr.Error(f'Unhandled Setting for {evt.target}')
+
+def clean_temp():
+    from ui.main import prepare_environment
+    
+    ui.globals.ui_input_thumbs.clear()
+    roop.globals.INPUT_FACESETS.clear()
+    roop.globals.TARGET_FACES.clear()
+    ui.globals.ui_target_thumbs = []
+    if not roop.globals.CFG.use_os_temp_folder:
+        clean_dir(os.environ["TEMP"])
+    prepare_environment()
+    gr.Info('Temp Files removed')
+    return None,None,None,None
+
+
+def apply_settings(themes, input_server_name, input_server_port, output_template):
+    from ui.main import show_msg
+
+    roop.globals.CFG.selected_theme = themes
+    roop.globals.CFG.server_name = input_server_name
+    roop.globals.CFG.server_port = input_server_port
+    roop.globals.CFG.output_template = output_template
+    roop.globals.CFG.save()
+    show_msg('Settings saved')
+
+
+def restart():
+    ui.globals.ui_restart_server = True