first draft

README.md

@@ -1,13 +1,13 @@
 ---
-title: Platzi Curso Streamlit Butterfly Gan
-emoji: 📉
-colorFrom: green
-colorTo: indigo
+title: Butterfly GAN
+emoji: 🦋
+colorFrom: blue
+colorTo: yellow
 sdk: streamlit
-sdk_version: 1.10.0
+sdk_version: 1.2.0
 app_file: app.py
-pinned: false
-license: mit
+pinned: true
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,178 @@
+# from distutils.command.build import build
+import streamlit as st # HF spaces at v1.2.0
+# from utils import load_model,generate,get_dataset,embed,make_meme
+from utils import load_model,generate
+# import streamlit.components.v1 as components
+# import io
+# import os
+
+# root_dir=os.path.dirname(os.path.abspath(__file__))
+# build_dir = os.path.join(root_dir, "custom_component/frontend/build")
+# _component_func = components.declare_component("release_butterflies", path=build_dir)
+# def release_butterflies(name, key=None):
+#     component_value = _component_func(name=name, key=key, default=0)
+#     return component_value
+
+
+## Configuración de nuestro demo
+st.title("ButterflyGAN")
+st.write("Light-GAN model trained on 1000 butterfly images taken from the Smithsonian Museum collection. \n \
+Based on [paper:](https://openreview.net/forum?id=1Fqg133qRaI) *Towards Faster and Stabilized GAN Training for High-fidelity Few-shot Image Synthesis*")
+
+st.sidebar.subheader("This butterfly does not exist! ")
+st.sidebar.image("assets/logo.png", width=200)
+st.sidebar.caption(f"[Model](https://huggingface.co/ceyda/butterfly_cropped_uniq1K_512) & [Dataset](https://huggingface.co/datasets/huggan/smithsonian_butterflies_subset) used")
+st.sidebar.caption(f"Made during the [huggan](https://github.com/huggingface/community-events) hackathon")
+st.sidebar.caption(f"Contributors:")
+st.sidebar.caption(f"[Ceyda Cinarel](https://github.com/cceyda) & [Jonathan Whitaker](https://datasciencecastnet.home.blog/)")
+
+
+## Cargamos modelo
+repo_id = 'ceyda/butterfly_cropped_uniq1K_512'
+version_modelo ='57d36a15546909557d9f967f47713236c8288838'
+modelo_gan = load_model(repo_id, version_modelo)
+
+# @st.experimental_singleton
+# def load_model_intocache(model_name,model_version):
+#     # model_name='ceyda/butterfly_512_base'
+#     gan = load_model(model_name,model_version)
+#     return gan
+
+# @st.experimental_singleton
+# def load_dataset():
+#     dataset=get_dataset()
+#     return dataset
+
+# @st.experimental_singleton
+# def load_variables():# Don't want to open read files over and over. not sure if it makes a diff
+#     latent_walk_code=open("assets/code_snippets/latent_walk.py").read()
+#     latent_walk_code_music=open("assets/code_snippets/latent_walk_music.py").read()
+#     return latent_walk_code,latent_walk_code_music
+
+# def img2download(image):
+#         imgByteArr = io.BytesIO()
+#         image.save(imgByteArr, format="JPEG")
+#         imgByteArr = imgByteArr.getvalue()
+#         return imgByteArr
+
+# model_name='ceyda/butterfly_cropped_uniq1K_512'
+# model_version='57d36a15546909557d9f967f47713236c8288838'
+# model_version=None
+
+
+# model=load_model_intocache(model_name,model_version)
+# dataset=loadk_dataset()
+# latent_walk_code, latent_walk_code_music=load_variables()
+
+# generate_menu="🦋 Make butterflies"
+# latent_walk_menu="🎧 Take a latent walk"
+# make_meme_menu="🐦 Make a meme"
+# mosaic_menu="👀 See the mosaic"
+# fun_menu="🙌 Release the butterflies"
+
+# screen = st.sidebar.radio("Pick a destination",[generate_menu,latent_walk_menu,make_meme_menu,mosaic_menu,fun_menu])
+
+## Generamos 4 mariposas
+n_mariposas =4 
+
+## Función que genera mariposas y lo guarda como un estado de la sesión
+def corre():
+    with st.spinner("Generando, espera un poco..."):
+        ims=generate(modelo_gan,n_mariposas)
+        st.session_state['ims'] = ims
+
+## Si no hay una imagen generada entonces generala
+if 'ims' not in st.session_state:
+    st.session_state['ims'] = None
+    corre()
+
+## ims contiene las imágenes generadas
+ims=st.session_state["ims"]
+
+## Si la usuaria da click en el botón entonces corremos la función genera()
+runb=st.button("Genera mariposas por favor", on_click=corre ,help="generated on the fly maybe slow")
+
+if ims is not None:
+    cols=st.columns(n_mariposas)
+    # picks=[False]*n_mariposas
+    for j,im in enumerate(ims):
+        i=j%n_mariposas
+        cols[i].image(im, use_column_width=True)
+        # picks[j]=cols[i].button("Find Nearest",key="pick_"+str(j))
+        
+    # if any(picks):
+    #     # st.write("Nearest butterflies:")
+    #     for i,pick in enumerate(picks):
+    #         if pick:
+    #             scores, retrieved_examples=dataset.get_nearest_examples('beit_embeddings', embed(ims[i]), k=5)
+    #             for r in retrieved_examples["image"]:
+    #                 cols[i].image(r, use_column_width=True)
+# st.write("Nearest neighbors found in the training set according to L2 distance on 'microsoft/beit-base-patch16-224' embeddings")
+# st.write(f"Latent dimension: {model.latent_dim}, image size:{model.image_size}")
+
+# elif screen == latent_walk_menu:
+
+#     st.write("Take a latent walk :musical_note: with cute butterflies")
+
+#     cols=st.columns(3)
+
+#     cols[0].caption("A regular walk (no music)")
+#     cols[0].video("assets/latent_walks/regular_walk.mp4")
+    
+#     cols[1].caption("Walk with music :butterfly:")
+#     cols[1].video("assets/latent_walks/walk_happyrock.mp4")
+#     cols[2].caption("Walk with music :butterfly:")
+#     cols[2].video("assets/latent_walks/walk_cute.mp4")
+    
+#     st.caption("Royalty Free Music from Bensound")
+#     st.write("🎧Did those butterflies seem to be dancing to the music?!Here is the secret:")
+#     with st.expander("See the Code Snippets"):
+#         st.write("A regular latent walk:")
+#         st.code(latent_walk_code, language='python')
+#         st.write(":musical_note: latent walk with music:")
+#         st.code(latent_walk_code_music, language='python')
+
+
+# elif screen == make_meme_menu:
+#     if "pigeon" not in st.session_state:
+#         st.session_state['pigeon'] = generate(model,1)[0]
+
+#     def get_pigeon():
+#         st.session_state['pigeon'] = generate(model,1)[0]
+    
+#     cols= st.columns(2)
+#     cols[0].button("change pigeon",on_click=get_pigeon)
+#     no_bg=cols[1].checkbox("Remove background?",True,help="Remove the background from pigeon")
+#     show_text=cols[1].checkbox("Show text?",True)
+    
+#     meme_text=st.text_input("Enter text","Is this a pigeon?")
+   
+    
+#     meme=make_meme(st.session_state['pigeon'],text=meme_text,show_text=show_text,remove_background=no_bg)
+#     st.image(meme)
+#     coly=st.columns(2)
+#     coly[0].download_button("Download", img2download(meme),mime="image/jpeg")
+#     coly[1].write("Made a cool one? [Share](https://twitter.com/intent/tweet?text=Check%20out%20the%20demo%20for%20Butterfly%20GAN%20%F0%9F%A6%8Bhttps%3A//huggingface.co/spaces/huggan/butterfly-gan%0Amade%20by%20%40ceyda_cinarel%20%26%20%40johnowhitaker%20) on Twitter")
+
+
+# elif screen == mosaic_menu:
+#     cols=st.columns(2)
+#     cols[0].markdown("These are all the butterflies in our [training set](https://huggingface.co/huggan/smithsonian_butterflies_subset)")
+#     cols[0].image("assets/train_data_mosaic_lowres.jpg")
+#     cols[0].write("🔎 view the high-res version [here](https://www.easyzoom.com/imageaccess/0c77e0e716f14ea7bc235447e5a4c397)")
+
+#     cols[1].markdown("These are the butterflies our model generated.")
+#     cols[1].image("assets/gen_mosaic_lowres.jpg")
+#     cols[1].write("🔎 view the high-res version [here](https://www.easyzoom.com/imageaccess/cbb04e81106c4c54a9d9f9dbfb236eab)")
+
+# elif screen == fun_menu:
+    
+#     cols=st.columns([1,2])
+#     cols[0].write("While working on this project")
+#     cols[0].image("assets/butterflies_everywhere.jpg")
+    
+#     with cols[1]:
+#         release_butterflies("Hello World")
+    
+
+## Feel free to add more & change stuff ^

packages.txt

@@ -0,0 +1 @@
+libgl1

requirements.txt

@@ -0,0 +1,5 @@
+git+https://github.com/huggingface/community-events.git@3fea10c5d5a50c69f509e34cd580fe9139905d04#egg=huggan
+transformers
+faiss-cpu
+paddlehub
+paddlepaddle

utils.py

@@ -0,0 +1,107 @@
+import torch
+from huggan.pytorch.lightweight_gan.lightweight_gan import LightweightGAN
+from datasets import load_dataset
+from PIL import Image
+import numpy as np
+import paddlehub as hub
+import random
+from PIL import ImageDraw,ImageFont
+
+import streamlit as st
+
+@st.experimental_singleton
+def load_bg_model():
+    bg_model = hub.Module(name='U2NetP', directory='assets/models/')
+    return bg_model
+
+
+bg_model = load_bg_model()
+def remove_bg(img):
+    result = bg_model.Segmentation(
+        images=[np.array(img)[:,:,::-1]],
+        paths=None,
+        batch_size=1,
+        input_size=320,
+        output_dir=None,
+        visualization=False)
+    output = result[0]
+    mask=Image.fromarray(output['mask'])
+    front=Image.fromarray(output['front'][:,:,::-1]).convert("RGBA")
+    front.putalpha(mask)
+    return front
+
+meme_template=Image.open("./assets/pigeon_meme.jpg").convert("RGBA")
+def make_meme(pigeon,text="Is this a pigeon?",show_text=True,remove_background=True):
+    
+    meme=meme_template.copy()
+    approx_butterfly_center=(850,30)
+
+    if remove_background:
+        pigeon=remove_bg(pigeon)
+        
+    else:
+        pigeon=Image.fromarray(pigeon).convert("RGBA")
+
+    random_rotate=random.randint(-30,30)
+    random_size=random.randint(150,200)
+    pigeon=pigeon.resize((random_size,random_size)).rotate(random_rotate,expand=True)
+
+    meme.alpha_composite(pigeon, approx_butterfly_center)
+
+    #ref: https://blog.lipsumarium.com/caption-memes-in-python/
+    def drawTextWithOutline(text, x, y):
+        draw.text((x-2, y-2), text,(0,0,0),font=font)
+        draw.text((x+2, y-2), text,(0,0,0),font=font)
+        draw.text((x+2, y+2), text,(0,0,0),font=font)
+        draw.text((x-2, y+2), text,(0,0,0),font=font)
+        draw.text((x, y), text, (255,255,255), font=font)
+
+    if show_text:
+        draw = ImageDraw.Draw(meme)
+        font_size=52
+        font = ImageFont.truetype("assets/impact.ttf", font_size)
+        w, h = draw.textsize(text, font) # measure the size the text will take
+        drawTextWithOutline(text, meme.width/2 - w/2, meme.height - font_size*2)
+    meme = meme.convert("RGB")    
+    return meme
+
+def get_train_data(dataset_name="huggan/smithsonian_butterflies_subset"):
+    dataset=load_dataset(dataset_name)
+    dataset=dataset.sort("sim_score")
+    return dataset["train"]
+
+from transformers import BeitFeatureExtractor, BeitForImageClassification
+emb_feature_extractor = BeitFeatureExtractor.from_pretrained('microsoft/beit-base-patch16-224')
+emb_model = BeitForImageClassification.from_pretrained('microsoft/beit-base-patch16-224')
+def embed(images):
+    inputs = emb_feature_extractor(images=images, return_tensors="pt")
+    outputs = emb_model(**inputs,output_hidden_states= True)
+    last_hidden=outputs.hidden_states[-1]
+    pooler=emb_model.base_model.pooler
+    final_emb=pooler(last_hidden).detach().numpy()
+    return final_emb    
+    
+def build_index():    
+    dataset=get_train_data()
+    ds_with_embeddings = dataset.map(lambda x: {"beit_embeddings":embed(x["image"])},batched=True,batch_size=20)
+    ds_with_embeddings.add_faiss_index(column='beit_embeddings')
+    ds_with_embeddings.save_faiss_index('beit_embeddings', 'beit_index.faiss')
+
+def get_dataset():
+    dataset=get_train_data()
+    dataset.load_faiss_index('beit_embeddings', 'beit_index.faiss')
+    return dataset
+
+def load_model(model_name='ceyda/butterfly_cropped_uniq1K_512',model_version=None):
+    gan = LightweightGAN.from_pretrained(model_name,version=model_version)
+    gan.eval()
+    return gan
+    
+def generate(gan,batch_size=1):
+    with torch.no_grad():
+        ims = gan.G(torch.randn(batch_size, gan.latent_dim)).clamp_(0., 1.)*255
+        ims = ims.permute(0,2,3,1).detach().cpu().numpy().astype(np.uint8)
+    return ims
+
+def interpolate():
+    pass