diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -1,743 +1,48 @@ -import streamlit as st -import streamlit_nested_expanders -import subprocess -import sys -import os -import gc -import pathlib, shutil -from os.path import exists as path_exists -import glob -from git.repo.base import Repo -import argparse -import random -import string -from io import BytesIO -from PIL import Image -import torch -import wget -from pathvalidate import sanitize_filename - -torch.cuda.empty_cache() - -st.set_page_config(layout="wide") -def gitclone(url): - res = subprocess.run(['git', 'clone', url], stdout=subprocess.PIPE).stdout.decode('utf-8') - print(res) - -@st.cache -def initial_clones(): - root_path = f'.' - model_path = f'.' - gitclone("https://github.com/MSFTserver/pytorch3d-lite.git") - gitclone("https://github.com/CompVis/taming-transformers.git") - gitclone("https://github.com/openai/CLIP.git") - gitclone("https://github.com/crowsonkb/guided-diffusion.git") - gitclone("https://github.com/assafshocher/ResizeRight.git") - gitclone("https://github.com/isl-org/MiDaS.git") - print(path_exists('MiDaS')) - if not path_exists(f'{root_path}/MiDaS/midas_utils.py'): - os.rename("MiDaS/utils.py", "MiDaS/midas_utils.py") - gitclone("https://github.com/CompVis/latent-diffusion.git") - gitclone("https://github.com/shariqfarooq123/AdaBins.git") - gitclone("https://github.com/alembics/disco-diffusion.git") - shutil.copyfile("disco-diffusion/disco_xform_utils.py", "disco_xform_utils.py") - wget.download("https://v-diffusion.s3.us-west-2.amazonaws.com/512x512_diffusion_uncond_finetune_008100.pt", model_path) - wget.download("https://v-diffusion.s3.us-west-2.amazonaws.com/secondary_model_imagenet_2.pth", model_path) - wget.download("https://cloudflare-ipfs.com/ipfs/Qmd2mMnDLWePKmgfS8m6ntAg4nhV5VkUyAydYBp8cWWeB7/AdaBins_nyu.pt", model_path) - torch.hub.download_url_to_file('https://heibox.uni-heidelberg.de/d/a7530b09fed84f80a887/files/?p=%2Fconfigs%2Fmodel.yaml&dl=1','vqgan_imagenet_f16_16384.yaml') - torch.hub.download_url_to_file('https://heibox.uni-heidelberg.de/d/a7530b09fed84f80a887/files/?p=%2Fckpts%2Flast.ckpt&dl=1','vqgan_imagenet_f16_16384.ckpt') - sys.path.append("./guided-diffusion") - sys.path.append("./latent-diffusion") - sys.path.append(".") - sys.path.append("./taming-transformers") - sys.path.append("./disco-diffusion") - sys.path.append("./AdaBins") - sys.path.append('./pytorch3d-lite') - -initial_clones() -# Horrible hacks below, Streamlit really makes your life hard when it comes to customzing -custom_css = """ - -""" - - -class DefaultPaths: - root_path = f"." - if not (path_exists(f"/content/drive/MyDrive/")): - is_drive = False - model_path = root_path - output_path = f"{root_path}/outputs" - else: - is_drive = True - drive_path = f"/content/drive/MyDrive/MindsEye" - model_path = f"{drive_path}/models" - output_path = f"{drive_path}/outputs" - -initial_load = st.empty() -initial_load.empty() - -st.write( - "

MindsEye beta ai art generator by @multimodalart

", - unsafe_allow_html=True, -) - -page_names = ["CLIP Guided Diffusion", "VQGAN+CLIP"] - -if "width" not in st.session_state: - st.session_state["width"] = 512 -if "height" not in st.session_state: - st.session_state["height"] = 512 -if "seed" not in st.session_state: - init_seed = int(random.randint(0, 2147483647)) - st.session_state.seed = init_seed -else: - init_seed = st.session_state.seed - -if "user_input" not in st.session_state: - st.session_state.user_input = "An açai bowl" -user_input = st.text_input( - "A text prompt to generate your image", - st.session_state.user_input, - help="The text you type here will be used for the AI to generate an image based on it. If you want multiple prompts you can separate them via a `|`, so if you want `sun|moon` it will try to utilize both. If you want that they are weighted differently, you can use `:` with the following syntax `sun:1|moon:2`, here `moon` will have 2x the weight of `sun`", -) - - -def add_to_prompt(text): - global user_input - st.session_state.user_input = user_input + " " + text - - -def dimensions_compatibility(type, after): - if type == "width": - dimension = st.session_state.width - elif type == "height": - dimension = st.session_state.height - if dimension % 64 != 0: - dimension = 64 * round(dimension / 64) - if type == "width": - st.session_state.width = dimension - elif type == "height": - st.session_state.height = dimension - after.info("We rounded your value to a multiple of 64") - - -enhancers = st.expander("Prompt enhancers (optional)") -with enhancers: - st.write( - "Adding enhancers to your prompts can produce very different results to the generated images, the few below are some some examples that can produce interesting results. Here you can learn more about Prompt Engineering", - unsafe_allow_html=True, - ) - with st.container(): - listofenhancers = [ - "trending on artstation", - "in ukiyo-e style", - ", oil on canvas", - "in chinese watercolor style", - "in art deco style", - "8k resolution", - "unreal engine 4k", - "cgsociety", - ", pencil sketch", - ", children's drawing", - "by Van Gogh", - "by Dan Mumford", - "1995", - ", an abstract sculpture", - " anime style", - "in soviet propaganda style", - ", persian miniature painting", - "lens flare", - "high quality", - ] - for enhancer in listofenhancers: - st.button(enhancer, on_click=add_to_prompt, args=(enhancer,)) - -# Soon add a media input for image/audio as prompts -# col2_input.write('and/or') -# user_input_image = col3_input.file_uploader("A media prompt (image or audio file)",type=["png","jpg","wav"],help="Media prompts only work with VQGAN-CLIP for now (Guided Diffusion coming soon)") - -# st.write('', - unsafe_allow_html=True, - ) - model.run_model(args, status, stoutput, DefaultPaths) - gray_during_execution.markdown( - '', - unsafe_allow_html=True, - ) - - -col_output1, col_output2 = st.columns(2) - - -def intermediary_frame_setup(seed): - if intermediary_frames: - intermediary_folder = f"{DefaultPaths.output_path}/{sanitize_filename(user_input)} [{sub_model}] {int(seed)}_frames" - if not path_exists(intermediary_folder): - os.makedirs(intermediary_folder) - update_every = how_many_frames - else: - intermediary_folder = None - update_every = 10 - return intermediary_folder, update_every - - -gray_during_execution = st.empty() -if submit: - meta_status = col_output1.empty() - status = col_output1.empty() - if uploaded_file is not None: - image_data = uploaded_file.read() - f = open(uploaded_file.name, "wb") - f.write(image_data) - f.close() - image_path = uploaded_file.name - else: - image_path = None - intermediary_folder, update_every = intermediary_frame_setup(seed) - if page == "VQGAN+CLIP": - args = argparse.Namespace( - prompt=user_input, - seed=int(seed), - sizex=width, - sizey=height, - flavor=flavor, - iterations=num_steps, - mse=True, - update=update_every, - template=template, - clip_model_1="ViT-B/32" if "ViT-B/32" in clip_model else None, - clip_model_2="ViT-B/16" if "ViT-B/16" in clip_model else None, - clip_model_3="ViT-L/14" if "ViT-L/14" in clip_model else None, - clip_model_4="RN50x64" if "RN50x64" in clip_model else None, - clip_model_5="RN50x16" if "RN50x16" in clip_model else None, - clip_model_6="RN50x4" if "RN50x4" in clip_model else None, - clip_model_7="RN50" if "RN50" in clip_model else None, - clip_model_8="RN101" if "RN101" in clip_model else None, - vqgan_model=vqgan_model, - seed_image=image_path, - image_file="progress.png", - frame_dir=intermediary_folder, - sub_model=sub_model, - ) - elif page == "CLIP Guided Diffusion": - args = argparse.Namespace( - prompt=user_input, - seed=int(seed), - iterations=num_steps, - update=update_every, - sizex=width, - sizey=height, - cutn=cutouts, - cutnbatches=cutout_batches, - tvscale=tv_scale, - rangescale=range_scale, - guidancescale=guidance_scale, - saturationscale=sat_scale, - skipseedtimesteps=skipseedtimesteps, - usevit32="ViT-B/32" in clip_model, - usevit16="ViT-B/16" in clip_model, - usevit14="ViT-L/14" in clip_model, - usern50x4="RN50x4" in clip_model, - usern50x16="RN50x16" in clip_model, - usern50x64="RN50x64" in clip_model, - usern50="RN50" in clip_model, - usern101="RN101" in clip_model, - useslipbase=False, - usesliplarge=False, - use256=False, - denoised=False, - useaugs=use_augs, - secondarymodel=secondary_model, - clampmax=clamp_max, - dango=True, - # ddim=False, - sampling_mode=sampling_mode, - eta=eta, - image_file="progress.png", - init_image=None, - clip_model=clip_model, - image_prompts=[], - seed_image=image_path, - initscale=initscale, - frame_dir=intermediary_folder, - animation_mode="None", - max_frames=10000, - angle="0:(0)", - zoom="0: (1), 10: (1.05)", - translation_x="0: (0)", - translation_y="0: (0)", - translation_z="0: (10.0)", - rotation_3d_x="0: (0)", - rotation_3d_y="0: (0)", - rotation_3d_z="0: (0)", - midas_weight=0.3, - near_plane=200, - far_plane=10000, - fov=40, - frames_scale=1500, - frames_skip_steps="60%", - turbo_mode=False, - turbo_steps=3, - perlin_init=0, - perlin_mode="mixed", - sub_model=sub_model, - ) - try: - if (how_many_runs) > 1: - if batch_folder: - DefaultPaths.output_path = f"{DefaultPaths.output_path}/{batch_folder}" - intermediary_folder, update_every = intermediary_frame_setup(args.seed) - for i in range(how_many_runs): - if how_many_runs > 1: - meta_status.write(f"Running batch, {i+1} out of {how_many_runs}") - if i > 0: - if randomize_seed: - args.seed = random.randint(0, 2147483647) - intermediary_folder, update_every = intermediary_frame_setup(args.seed) - args.frame_dir = intermediary_folder - run_internal(args, status, col_output1, gray_during_execution) - gc.collect() - torch.cuda.empty_cache() - except st.script_runner.StopException as e: - status.empty() - gc.collect() - torch.cuda.empty_cache() - gray_during_execution.markdown( - '', - unsafe_allow_html=True, - ) - pass - init_seed = int(random.randint(0, 2147483647)) - st.session_state.seed = init_seed - meta_status.empty() - st.experimental_rerun() - -with col_output2: - gallery_text_area = st.empty() - gallery_image_area = st.empty() - if DefaultPaths.is_drive: - output_folder = f"{DefaultPaths.drive_path}/outputs" - if os.listdir(output_folder): - files_path = os.path.join(output_folder, "*.png") - files = sorted(glob.iglob(files_path), key=os.path.getctime, reverse=True) - gallery_text_area.write("Welcome back! Your last creation:") - gallery_image_area.image(Image.open(files[0])) - st.write( - f'
View your gallery on Google Drive
We do not collect prompts or results. Your creations don\'t belong to MindsEye. Read our FAQ.
Feel free to reference #MindsEye and tag @multimodalart when sharing your creations if you wish
', - unsafe_allow_html=True, - ) - else: - st.write( - f'
We do not collect prompts or results. Your creations don\'t belong to MindsEye. Read our FAQ.
Feel free to reference #MindsEye and tag @multimodalart when sharing your creations if you wish
', - unsafe_allow_html=True, - ) - if os.path.exists("progress.png"): - gallery_text_area.write("Your last creation:") - gallery_image_area.image(Image.open("progress.png")) - -footer = """ -
-

MindsEye beta by @multimodalart
-Disco Diffusion v5 model by @somnai_dreams and @gandamu, based on the foundational work of @RiversHaveWings, with modifications by @danielrussruss, Dango233, Chigozie Nri, @softologyComAu and others.
Hypertron v2 VQGAN model by Philipuss adapted from @RiversHaveWings with modifications by @jbusted1, @softologyComAu and others. Original GAN+CLIP by @advadnoun. CLIP and Guided Diffusion were originally released by OpenAI. VQGAN was released by CompVis Heidelberg
- application made with streamlit -

-""" -st.markdown(footer, unsafe_allow_html=True) +from turtle import width +import gradio as gr +import inspect +latent = gr.Interface.load("spaces/multimodalart/latentdiffusion") +rudalle = gr.Interface.load("spaces/multimodalart/rudalle") +guided = gr.Interface.load("spaces/EleutherAI/clip-guided-diffusion") +print(guided) +def text2image_latent(text,steps,width,height,images,diversity): + image = latent(text, steps, width, height, images, diversity)[0] + print(image) + return(image) + #return gr.processing_utils.decode_base64_to_image(image) + +def text2image_rudalle(text,aspect,model): + image = rudalle(text,aspect,model) + print(image) + return(image) + +def text2image_guided(text): + image = guided(text)[0] + return(image) + + +block = gr.Blocks() + +with block: + text = gr.inputs.Textbox(placeholder="Try writing something..") + with gr.Tab("Latent Diffusion"): + steps = gr.inputs.Slider(label="Steps - more steps can increase quality but will take longer to generate",default=45,maximum=50,minimum=1,step=1) + width = gr.inputs.Radio(label="Width", choices=[32,64,128,256],default=256) + height = gr.inputs.Radio(label="Height", choices=[32,64,128,256],default=256) + images = gr.inputs.Slider(label="Images - How many images you wish to generate", default=2, step=1, minimum=1, maximum=4) + diversity = gr.inputs.Slider(label="Diversity scale - How different from one another you wish the images to be",default=5.0, minimum=1.0, maximum=15.0) + get_image_latent = gr.Button("Generate Image") + with gr.Tab("ruDALLE"): + aspect = gr.inputs.Radio(label="Aspect Ratio", choices=["Square", "Horizontal", "Vertical"],default="Square") + model = gr.inputs.Dropdown(label="Model", choices=["Surrealism","Realism", "Emoji"], default="Surrealism") + get_image_rudalle = gr.Button("Generate Image") + with gr.Tab("Guided Diffusion"): + get_image_guided = gr.Button("Generate Image") + with gr.Column(): + with gr.Row(): + image = gr.outputs.Image() + + get_image_latent.click(text2image_latent, inputs=[text,steps,width,height,images,diversity], outputs=image) + get_image_rudalle.click(text2image_rudalle, inputs=[text,aspect,model], outputs=image) + get_image_guided.click(text2image_guided, inputs=text, outputs=image) +block.launch()