safi842 commited on
Commit
6a7d732
1 Parent(s): 5221ac5

UI/Perf fixes

Browse files
Files changed (7) hide show
  1. README.md +1 -0
  2. app.py +12 -256
  3. pages/Fusion Fashion.py +285 -0
  4. pages/Style One.py +259 -0
  5. pics/logo.jpeg +0 -0
  6. pics/logo_small.jpeg +0 -0
  7. requirements.txt +2 -1
README.md CHANGED
@@ -8,6 +8,7 @@ sdk_version: 1.19.0
8
  app_file: app.py
9
  pinned: true
10
  license: afl-3.0
 
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
8
  app_file: app.py
9
  pinned: true
10
  license: afl-3.0
11
+ duplicated_from: safi842/FashionGen
12
  ---
13
 
14
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py CHANGED
@@ -1,265 +1,21 @@
1
- import random
2
  import streamlit as st
3
- import torch
4
- import PIL
5
- import numpy as np
6
- from PIL import Image
7
- import imageio
8
- from models import get_instrumented_model
9
- from decomposition import get_or_compute
10
- from config import Config
11
- from skimage import img_as_ubyte
12
- import clip
13
- from torchvision.transforms import Resize, Normalize, Compose, CenterCrop
14
- from torch.optim import Adam
15
- from stqdm import stqdm
16
 
17
- torch.set_num_threads(8)
18
 
19
- # Speed up computation
20
- torch.autograd.set_grad_enabled(True)
21
- #torch.backends.cudnn.benchmark = True
22
-
23
- # Specify model to use
24
- config = Config(
25
- model='StyleGAN2',
26
- layer='style',
27
- output_class= 'lookbook',
28
- components=80,
29
- use_w=True,
30
- batch_size=5_000, # style layer quite small
31
  )
32
 
33
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
34
-
35
- preprocess = Compose([
36
- Resize(224),
37
- CenterCrop(224),
38
- Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711)),
39
- ])
40
-
41
- @st.cache_data
42
- def clip_optimized_latent(text, seed, iterations=25, lr=1e-2):
43
- seed = int(seed)
44
- text_input = clip.tokenize([text]).to(device)
45
-
46
- # Initialize a random latent vector
47
- latent_vector = model.sample_latent(1,seed=seed).detach()
48
- latent_vector.requires_grad = True
49
- latent_vector = [latent_vector]*model.get_max_latents()
50
- params = [torch.nn.Parameter(latent_vector[i], requires_grad=True) for i in range(len(latent_vector))]
51
- optimizer = Adam(params, lr=lr)
52
-
53
- with torch.no_grad():
54
- text_features = clip_model.encode_text(text_input)
55
-
56
- #pbar = tqdm(range(iterations), dynamic_ncols=True)
57
-
58
- for iteration in stqdm(range(iterations)):
59
- optimizer.zero_grad()
60
-
61
- # Generate an image from the latent vector
62
- image = model.sample(params)
63
- image = image.to(device)
64
-
65
- # Preprocess the image for the CLIP model
66
- image = preprocess(image)
67
- #image = clip_preprocess(Image.fromarray((image_np * 255).astype(np.uint8))).unsqueeze(0).to(device)
68
-
69
- # Extract features from the image
70
- image_features = clip_model.encode_image(image)
71
-
72
- # Calculate the loss and backpropagate
73
- loss = -torch.cosine_similarity(text_features, image_features).mean()
74
- loss.backward()
75
- optimizer.step()
76
-
77
- #pbar.set_description(f"Loss: {loss.item()}") # Update the progress bar to show the current loss
78
- w = [param.detach().cpu().numpy() for param in params]
79
-
80
- return w
81
-
82
- def mix_w(w1, w2, content, style):
83
- for i in range(0,5):
84
- w2[i] = w1[i] * (1 - content) + w2[i] * content
85
-
86
- for i in range(5, 16):
87
- w2[i] = w1[i] * (1 - style) + w2[i] * style
88
-
89
- return w2
90
-
91
- def display_sample_pytorch(seed, truncation, directions, distances, scale, start, end, w=None, disp=True, save=None, noise_spec=None):
92
- # blockPrint()
93
- model.truncation = truncation
94
- if w is None:
95
- w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
96
- w = [w]*model.get_max_latents() # one per layer
97
- else:
98
- w_numpy = [x.detach().numpy() for x in w]
99
- w = [np.expand_dims(x, 0) for x in w_numpy]
100
- #w = [x.unsqueeze(0) for x in w]
101
-
102
-
103
- for l in range(start, end):
104
- for i in range(len(directions)):
105
- w[l] = w[l] + directions[i] * distances[i] * scale
106
-
107
- torch.cuda.empty_cache()
108
- #save image and display
109
- out = model.sample(w)
110
- out = out.permute(0, 2, 3, 1).cpu().detach().numpy()
111
- out = np.clip(out, 0.0, 1.0).squeeze()
112
-
113
- final_im = Image.fromarray((out * 255).astype(np.uint8)).resize((500,500),Image.LANCZOS)
114
-
115
-
116
- if save is not None:
117
- if disp == False:
118
- print(save)
119
- final_im.save(f'out/{seed}_{save:05}.png')
120
- if disp:
121
- display(final_im)
122
-
123
- return final_im
124
-
125
- ## Generate image for app
126
- def generate_image(content, style, truncation, c0, c1, c2, c3, c4, c5, c6, start_layer, end_layer,w1,w2):
127
-
128
- scale = 1
129
- params = {'c0': c0,
130
- 'c1': c1,
131
- 'c2': c2,
132
- 'c3': c3,
133
- 'c4': c4,
134
- 'c5': c5,
135
- 'c6': c6}
136
-
137
- param_indexes = {'c0': 0,
138
- 'c1': 1,
139
- 'c2': 2,
140
- 'c3': 3,
141
- 'c4': 4,
142
- 'c5': 5,
143
- 'c6': 6}
144
-
145
- directions = []
146
- distances = []
147
- for k, v in params.items():
148
- directions.append(latent_dirs[param_indexes[k]])
149
- distances.append(v)
150
-
151
- if w1 is not None and w2 is not None:
152
- w1 = [torch.from_numpy(x).to(device) for x in w1]
153
- w2 = [torch.from_numpy(x).to(device) for x in w2]
154
-
155
-
156
- #w1 = clip_optimized_latent(text1, seed1, iters)
157
- im1 = model.sample(w1)
158
- im1_np = im1.permute(0, 2, 3, 1).cpu().detach().numpy()
159
- im1_np = np.clip(im1_np, 0.0, 1.0).squeeze()
160
-
161
- #w2 = clip_optimized_latent(text2, seed2, iters)
162
- im2 = model.sample(w2)
163
- im2_np = im2.permute(0, 2, 3, 1).cpu().detach().numpy()
164
- im2_np = np.clip(im2_np, 0.0, 1.0).squeeze()
165
-
166
- combined_im = np.concatenate([im1_np, im2_np], axis=1)
167
- input_im = Image.fromarray((combined_im * 255).astype(np.uint8))
168
-
169
-
170
- mixed_w = mix_w(w1, w2, content, style)
171
- return input_im, display_sample_pytorch(seed1, truncation, directions, distances, scale, int(start_layer), int(end_layer), w=mixed_w, disp=False)
172
-
173
-
174
- # Streamlit app title
175
- st.title("FashionGen Demo - AI assisted fashion design")
176
- """This application employs the StyleGAN framework, CLIP and GANSpace exploration techniques to synthesize images of garments from textual inputs. With training based on the comprehensive LookBook dataset, it supports an efficient fashion design process by transforming text into visual concepts, showcasing the practical application of Generative Adversarial Networks (GANs) in the realm of creative design."""
177
-
178
- @st.cache_resource
179
- def load_model():
180
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
181
- # Load the pre-trained CLIP model
182
- clip_model, clip_preprocess = clip.load("ViT-B/32", device=device)
183
- inst = get_instrumented_model(config.model, config.output_class,
184
- config.layer, torch.device('cpu'), use_w=config.use_w)
185
- return clip_model, inst
186
-
187
- # Then, to load your models, call this function:
188
- clip_model, inst = load_model()
189
- model = inst.model
190
-
191
-
192
- path_to_components = get_or_compute(config, inst)
193
- comps = np.load(path_to_components)
194
- lst = comps.files
195
- latent_dirs = []
196
- latent_stdevs = []
197
-
198
- load_activations = False
199
-
200
- for item in lst:
201
- if load_activations:
202
- if item == 'act_comp':
203
- for i in range(comps[item].shape[0]):
204
- latent_dirs.append(comps[item][i])
205
- if item == 'act_stdev':
206
- for i in range(comps[item].shape[0]):
207
- latent_stdevs.append(comps[item][i])
208
- else:
209
- if item == 'lat_comp':
210
- for i in range(comps[item].shape[0]):
211
- latent_dirs.append(comps[item][i])
212
- if item == 'lat_stdev':
213
- for i in range(comps[item].shape[0]):
214
- latent_stdevs.append(comps[item][i])
215
-
216
- ## Side bar texts
217
- st.sidebar.title('Tuning Parameters')
218
- st.sidebar.subheader('(CLIP + GANSpace)')
219
-
220
-
221
- # Create UI widgets
222
-
223
- if 'seed1' not in st.session_state and 'seed2' not in st.session_state:
224
- st.session_state['seed1'] = random.randint(1, 1000)
225
- st.session_state['seed2'] = random.randint(1, 1000)
226
- seed1 = st.sidebar.number_input("Seed 1", value= st.session_state['seed1'])
227
- seed2 = st.sidebar.number_input("Seed 2", value= st.session_state['seed2'])
228
- text1 = st.sidebar.text_input("Text Description 1")
229
- text2 = st.sidebar.text_input("Text Description 2")
230
- iters = st.sidebar.number_input("Iterations for CLIP Optimization", value = 25)
231
- submit_button = st.sidebar.button("Submit")
232
- content = st.sidebar.slider("Structural Composition", min_value=0.0, max_value=1.0, value=0.5)
233
- style = st.sidebar.slider("Style", min_value=0.0, max_value=1.0, value=0.5)
234
- truncation = st.sidebar.slider("Dimensional Scaling", min_value=0.0, max_value=1.0, value=0.5)
235
-
236
- slider_min_val = -20
237
- slider_max_val = 20
238
- slider_step = 1
239
-
240
- c0 = st.sidebar.slider("Sleeve Size Scaling", min_value=slider_min_val, max_value=slider_max_val, value=0)
241
- c1 = st.sidebar.slider("Jacket Features", min_value=slider_min_val, max_value=slider_max_val, value=0)
242
- c2 = st.sidebar.slider("Women's Overcoat", min_value=slider_min_val, max_value=slider_max_val, value=0)
243
- c3 = st.sidebar.slider("Coat", min_value=slider_min_val, max_value=slider_max_val, value=0)
244
- c4 = st.sidebar.slider("Graphic Elements", min_value=slider_min_val, max_value=slider_max_val, value=0)
245
- c5 = st.sidebar.slider("Darker Color", min_value=slider_min_val, max_value=slider_max_val, value=0)
246
- c6 = st.sidebar.slider("Modest Neckline", min_value=slider_min_val, max_value=slider_max_val, value=0)
247
- start_layer = st.sidebar.number_input("Start Layer", value=0)
248
- end_layer = st.sidebar.number_input("End Layer", value=14)
249
-
250
 
 
251
 
252
- if submit_button: # Execute when the submit button is pressed
253
- w1 = clip_optimized_latent(text1, seed1, iters)
254
- st.session_state['w1-np'] = w1
255
- w2 = clip_optimized_latent(text2, seed2, iters)
256
- st.session_state['w2-np'] = w2
257
 
258
- try:
259
- input_im, output_im = generate_image(content, style, truncation, c0, c1, c2, c3, c4, c5, c6, start_layer, end_layer,st.session_state['w1-np'],st.session_state['w2-np'])
260
- st.image(input_im, caption="Input Image")
261
- st.image(output_im, caption="Output Image")
262
- except:
263
- pass
264
 
265
-
 
 
 
 
1
  import streamlit as st
2
+ #from streamlit_extras.app_logo import add_logo
 
 
 
 
 
 
 
 
 
 
 
 
3
 
 
4
 
5
+ st.set_page_config(
6
+ page_title="FashionGen",
7
+ page_icon="👗",
 
 
 
 
 
 
 
 
 
8
  )
9
 
10
+ #st.title("FashionGen")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
+ #add_logo("./pics/logo_small.jpeg", height=20)
13
 
14
+ st.image('./pics/logo.jpeg')
15
+ '''#### About:'''
16
+ '''This application employs the StyleGAN framework, CLIP and GANSpace exploration techniques to synthesize images of garments from textual inputs. With training based on the comprehensive LookBook dataset, it supports an efficient fashion design process by transforming text into visual concepts, showcasing the practical application of Generative Adversarial Networks (GANs) in the realm of creative design.'''
 
 
17
 
 
 
 
 
 
 
18
 
19
+ ''' There are two modes of image generatation: \n
20
+ **Fashion Fusion:** Takes two descriptions and generates two designs whose styles features can be combined and edited. \n
21
+ **Style One:** Takes a single description and generates one design whose style features can be edited.'''
pages/Fusion Fashion.py ADDED
@@ -0,0 +1,285 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+ import streamlit as st
3
+ import torch
4
+ import PIL
5
+ import numpy as np
6
+ from PIL import Image
7
+ import imageio
8
+ from models import get_instrumented_model
9
+ from decomposition import get_or_compute
10
+ from config import Config
11
+ from skimage import img_as_ubyte
12
+ import clip
13
+ from torchvision.transforms import Resize, Normalize, Compose, CenterCrop
14
+ from torch.optim import Adam
15
+ from stqdm import stqdm
16
+
17
+
18
+ st.set_page_config(
19
+ page_title="Fusion Fashion",
20
+ page_icon="👗",
21
+ )
22
+
23
+ #torch.set_num_threads(8)
24
+
25
+ # Speed up computation
26
+ torch.autograd.set_grad_enabled(True)
27
+ torch.backends.cudnn.benchmark = True
28
+
29
+ # Specify model to use
30
+ config = Config(
31
+ model='StyleGAN2',
32
+ layer='style',
33
+ output_class= 'lookbook',
34
+ components=80,
35
+ use_w=True,
36
+ batch_size=5_000, # style layer quite small
37
+ )
38
+
39
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
40
+
41
+ preprocess = Compose([
42
+ Resize(224),
43
+ CenterCrop(224),
44
+ Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711)),
45
+ ])
46
+
47
+ @st.cache_data
48
+ def clip_optimized_latent(text, seed, iterations=25, lr=1e-2):
49
+ seed = int(seed)
50
+ text_input = clip.tokenize([text]).to(device)
51
+
52
+ # Initialize a random latent vector
53
+ latent_vector = model.sample_latent(1,seed=seed).detach().to(device)
54
+ latent_vector.requires_grad = True
55
+ latent_vector = [latent_vector]*model.get_max_latents()
56
+ params = [torch.nn.Parameter(latent_vector[i], requires_grad=True) for i in range(len(latent_vector))]
57
+ optimizer = Adam(params, lr=lr, betas=(0.9, 0.999))
58
+
59
+ #with torch.no_grad():
60
+ # text_features = clip_model.encode_text(text_input)
61
+
62
+ #pbar = tqdm(range(iterations), dynamic_ncols=True)
63
+
64
+ for iteration in stqdm(range(iterations)):
65
+ optimizer.zero_grad()
66
+
67
+ # Generate an image from the latent vector
68
+ image = model.sample(params)
69
+ image = image.to(device)
70
+
71
+ # Preprocess the image for the CLIP model
72
+ image = preprocess(image)
73
+ #image = clip_preprocess(Image.fromarray((image_np * 255).astype(np.uint8))).unsqueeze(0).to(device)
74
+
75
+ # Extract features from the image
76
+ #image_features = clip_model.encode_image(image)
77
+
78
+ # Calculate the loss and backpropagate
79
+ loss = 1 - clip_model(image, text_input)[0] / 100
80
+ #loss = -torch.cosine_similarity(text_features, image_features).mean()
81
+ loss.backward()
82
+ optimizer.step()
83
+
84
+ #pbar.set_description(f"Loss: {loss.item()}") # Update the progress bar to show the current loss
85
+ w = [param.detach().cpu().numpy() for param in params]
86
+
87
+ return w
88
+
89
+ def mix_w(w1, w2, content, style):
90
+ for i in range(0,5):
91
+ w2[i] = w1[i] * (1 - content) + w2[i] * content
92
+
93
+ for i in range(5, 16):
94
+ w2[i] = w1[i] * (1 - style) + w2[i] * style
95
+
96
+ return w2
97
+
98
+ def display_sample_pytorch(seed, truncation, directions, distances, scale, start, end, w=None, disp=True, save=None, noise_spec=None):
99
+ # blockPrint()
100
+ model.truncation = truncation
101
+ if w is None:
102
+ w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
103
+ w = [w]*model.get_max_latents() # one per layer
104
+ else:
105
+ w_numpy = [x.cpu().detach().numpy() for x in w]
106
+ w = [np.expand_dims(x, 0) for x in w_numpy]
107
+ #w = [x.unsqueeze(0) for x in w]
108
+
109
+
110
+ for l in range(start, end):
111
+ for i in range(len(directions)):
112
+ w[l] = w[l] + directions[i] * distances[i] * scale
113
+
114
+ w = [torch.from_numpy(x).to(device) for x in w]
115
+ torch.cuda.empty_cache()
116
+ #save image and display
117
+ out = model.sample(w)
118
+ out = out.permute(0, 2, 3, 1).cpu().detach().numpy()
119
+ out = np.clip(out, 0.0, 1.0).squeeze()
120
+
121
+ final_im = Image.fromarray((out * 255).astype(np.uint8)).resize((500,500),Image.LANCZOS)
122
+
123
+
124
+ if save is not None:
125
+ if disp == False:
126
+ print(save)
127
+ final_im.save(f'out/{seed}_{save:05}.png')
128
+ if disp:
129
+ display(final_im)
130
+
131
+ return final_im
132
+
133
+ ## Generate image for app
134
+ def generate_image(content, style, truncation, c0, c1, c2, c3, c4, c5, c6, start_layer, end_layer,w1,w2):
135
+
136
+ scale = 1
137
+ params = {'c0': c0,
138
+ 'c1': c1,
139
+ 'c2': c2,
140
+ 'c3': c3,
141
+ 'c4': c4,
142
+ 'c5': c5,
143
+ 'c6': c6}
144
+
145
+ param_indexes = {'c0': 0,
146
+ 'c1': 1,
147
+ 'c2': 2,
148
+ 'c3': 3,
149
+ 'c4': 4,
150
+ 'c5': 5,
151
+ 'c6': 6}
152
+
153
+ directions = []
154
+ distances = []
155
+ for k, v in params.items():
156
+ directions.append(latent_dirs[param_indexes[k]])
157
+ distances.append(v)
158
+
159
+ if w1 is not None and w2 is not None:
160
+ w1 = [torch.from_numpy(x).to(device) for x in w1]
161
+ w2 = [torch.from_numpy(x).to(device) for x in w2]
162
+
163
+
164
+ #w1 = clip_optimized_latent(text1, seed1, iters)
165
+ im1 = model.sample(w1)
166
+ im1_np = im1.permute(0, 2, 3, 1).cpu().detach().numpy()
167
+ im1_np = np.clip(im1_np, 0.0, 1.0).squeeze()
168
+
169
+ #w2 = clip_optimized_latent(text2, seed2, iters)
170
+ im2 = model.sample(w2)
171
+ im2_np = im2.permute(0, 2, 3, 1).cpu().detach().numpy()
172
+ im2_np = np.clip(im2_np, 0.0, 1.0).squeeze()
173
+
174
+ combined_im = np.concatenate([im1_np, im2_np], axis=1)
175
+ input_im = Image.fromarray((combined_im * 255).astype(np.uint8))
176
+
177
+
178
+ mixed_w = mix_w(w1, w2, content, style)
179
+ return input_im, display_sample_pytorch(seed1, truncation, directions, distances, scale, int(start_layer), int(end_layer), w=mixed_w, disp=False)
180
+
181
+
182
+ # Streamlit app title
183
+ st.image('./pics/logo.jpeg')
184
+ '''## Fusion Fashion'''
185
+
186
+ @st.cache_resource
187
+ def load_model():
188
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
189
+ # Load the pre-trained CLIP model
190
+ clip_model, clip_preprocess = clip.load("ViT-B/32", device=device)
191
+ inst = get_instrumented_model(config.model, config.output_class,
192
+ config.layer, device, use_w=config.use_w)
193
+ return clip_model, inst
194
+
195
+ # Then, to load your models, call this function:
196
+ clip_model, inst = load_model()
197
+ model = inst.model
198
+
199
+
200
+ path_to_components = get_or_compute(config, inst)
201
+ comps = np.load(path_to_components)
202
+ lst = comps.files
203
+ latent_dirs = []
204
+ latent_stdevs = []
205
+
206
+ load_activations = False
207
+
208
+ for item in lst:
209
+ if load_activations:
210
+ if item == 'act_comp':
211
+ for i in range(comps[item].shape[0]):
212
+ latent_dirs.append(comps[item][i])
213
+ if item == 'act_stdev':
214
+ for i in range(comps[item].shape[0]):
215
+ latent_stdevs.append(comps[item][i])
216
+ else:
217
+ if item == 'lat_comp':
218
+ for i in range(comps[item].shape[0]):
219
+ latent_dirs.append(comps[item][i])
220
+ if item == 'lat_stdev':
221
+ for i in range(comps[item].shape[0]):
222
+ latent_stdevs.append(comps[item][i])
223
+
224
+ ## Side bar texts
225
+ st.sidebar.title('Customization Options')
226
+
227
+
228
+ # Create UI widgets
229
+ text1 = st.sidebar.text_input("Style Specs 1", help = "Provide a clear and concise description of the design you wish to generate. This helps the app understand your preferences and create a customized design that matches your vision.")
230
+ text2 = st.sidebar.text_input("Style Specs 2", help = "Provide a clear and concise description of the design you wish to generate. This helps the app understand your preferences and create a customized design that matches your vision.")
231
+ if 'seed1' not in st.session_state and 'seed2' not in st.session_state:
232
+ #st.session_state['seed1'] = random.randint(1, 1000)
233
+ st.session_state['seed1'] = 3
234
+ #st.session_state['seed2'] = random.randint(1, 1000)
235
+ st.session_state['seed2'] = 200
236
+
237
+ with st.sidebar.expander("Advanced"):
238
+ seed1 = st.number_input("ID 1", value= st.session_state['seed1'], help = "Capture this unique id to reproduce the exact same result later.")
239
+ seed2 = st.number_input("ID 2", value= st.session_state['seed2'], help = "Capture this unique id to reproduce the exact same result later.")
240
+
241
+ st.session_state['seed1'] = seed1
242
+ st.session_state['seed2'] = seed2
243
+ iters = st.number_input("Cycles", value = 25, help = "Increase the sensitivity of the algorithm to find the design matching the style description. Higher values might enhance the accuracy but may lead to slower loading times")
244
+
245
+ submit_button = st.sidebar.button("Discover")
246
+ content = st.sidebar.slider("Design Frame", min_value=0.0, max_value=1.0, value=0.5, help = "Increasing it makes the structure similar to the image on the right and decreasing it will make the structure similar to the image on the left")
247
+ style = st.sidebar.slider("Style Composition", min_value=0.0, max_value=1.0, value=0.5, help = "Increasing it makes the style/pattern similar to the image on the right and decreasing it will make the style/pattern similar to the image on the left")
248
+ truncation = 0.5
249
+ #truncation = st.sidebar.slider("Dimensional Scaling", min_value=0.0, max_value=1.0, value=0.5)
250
+
251
+ slider_min_val = -20
252
+ slider_max_val = 20
253
+ slider_step = 1
254
+
255
+ c0 = st.sidebar.slider("Sleeve Size Scaling", min_value=slider_min_val, max_value=slider_max_val, value=0, help="Adjust the scaling of sleeve sizes. Increase to make sleeve sizes appear larger, and decrease to make them appear smaller.")
256
+ c1 = st.sidebar.slider("Jacket Features", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Control the prominence of jacket features. Increasing this value will make the features more pronounced, while decreasing it will make them less noticeable")
257
+ c2 = st.sidebar.slider("Women's Overcoat", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Modify the dominance of the women's overcoat style. Increase the value to enhance its prominence, and decrease it to reduce its impact.")
258
+ c3 = st.sidebar.slider("Coat", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Control the prominence of coat features. Increasing this value will make the features more pronounced, while decreasing it will make them less noticeable")
259
+ c4 = st.sidebar.slider("Graphic Elements", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Fine-tune the visibility of graphic elements. Increasing this value will make the graphics more prominent, while decreasing it will make them less visible.")
260
+ c5 = st.sidebar.slider("Darker Color", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Adjust the intensity of the color tones towards darker shades. Increasing this value will make the colors appear deeper, while decreasing it will lighten the overall color palette.")
261
+ c6 = st.sidebar.slider("Neckline", min_value=slider_min_val, max_value=slider_max_val, value=0,help = "Control the emphasis on the neckline of the garment. Increase to highlight the neckline, and decrease to downplay its prominence.")
262
+ start_layer = 0
263
+ end_layer = 14
264
+ #start_layer = st.sidebar.number_input("Start Layer", value=0)
265
+ #end_layer = st.sidebar.number_input("End Layer", value=14)
266
+
267
+ # if 'w1-np' not in st.session_state:
268
+ # st.session_state['w1-np'] = None
269
+
270
+ # if 'w2-np' not in st.session_state:
271
+ # st.session_state['w2-np'] = None
272
+
273
+
274
+ if submit_button: # Execute when the submit button is pressed
275
+ w1 = clip_optimized_latent(text1, seed1, iters)
276
+ st.session_state['w1-np'] = w1
277
+ w2 = clip_optimized_latent(text2, seed2, iters)
278
+ st.session_state['w2-np'] = w2
279
+
280
+ try:
281
+ input_im, output_im = generate_image(content, style, truncation, c0, c1, c2, c3, c4, c5, c6, start_layer, end_layer,st.session_state['w1-np'],st.session_state['w2-np'])
282
+ st.image(input_im, caption="Input Image")
283
+ st.image(output_im, caption="Output Image")
284
+ except:
285
+ pass
pages/Style One.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+ import streamlit as st
3
+ import torch
4
+ import PIL
5
+ import numpy as np
6
+ from PIL import Image
7
+ import imageio
8
+ from models import get_instrumented_model
9
+ from decomposition import get_or_compute
10
+ from config import Config
11
+ from skimage import img_as_ubyte
12
+ import clip
13
+ from torchvision.transforms import Resize, Normalize, Compose, CenterCrop
14
+ from torch.optim import Adam
15
+ from stqdm import stqdm
16
+
17
+
18
+ st.set_page_config(
19
+ page_title="Style One",
20
+ page_icon="👗",
21
+ )
22
+
23
+ #torch.set_num_threads(8)
24
+
25
+ # Speed up computation
26
+ torch.autograd.set_grad_enabled(True)
27
+ torch.backends.cudnn.benchmark = True
28
+
29
+ # Specify model to use
30
+ config = Config(
31
+ model='StyleGAN2',
32
+ layer='style',
33
+ output_class= 'lookbook',
34
+ components=80,
35
+ use_w=True,
36
+ batch_size=5_000, # style layer quite small
37
+ )
38
+
39
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
40
+
41
+ preprocess = Compose([
42
+ Resize(224),
43
+ CenterCrop(224),
44
+ Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711)),
45
+ ])
46
+
47
+ @st.cache_data
48
+ def clip_optimized_latent(text, seed, iterations=25, lr=1e-2):
49
+ seed = int(seed)
50
+ text_input = clip.tokenize([text]).to(device)
51
+
52
+ # Initialize a random latent vector
53
+ latent_vector = model.sample_latent(1,seed=seed).detach().to(device)
54
+ latent_vector.requires_grad = True
55
+ latent_vector = [latent_vector]*model.get_max_latents()
56
+ params = [torch.nn.Parameter(latent_vector[i], requires_grad=True) for i in range(len(latent_vector))]
57
+ optimizer = Adam(params, lr=lr, betas=(0.9, 0.999))
58
+
59
+ #with torch.no_grad():
60
+ # text_features = clip_model.encode_text(text_input)
61
+
62
+ #pbar = tqdm(range(iterations), dynamic_ncols=True)
63
+
64
+ for iteration in stqdm(range(iterations)):
65
+ optimizer.zero_grad()
66
+
67
+ # Generate an image from the latent vector
68
+ image = model.sample(params)
69
+ image = image.to(device)
70
+
71
+ # Preprocess the image for the CLIP model
72
+ image = preprocess(image)
73
+ #image = clip_preprocess(Image.fromarray((image_np * 255).astype(np.uint8))).unsqueeze(0).to(device)
74
+
75
+ # Extract features from the image
76
+ #image_features = clip_model.encode_image(image)
77
+
78
+ # Calculate the loss and backpropagate
79
+ loss = 1 - clip_model(image, text_input)[0] / 100
80
+ #loss = -torch.cosine_similarity(text_features, image_features).mean()
81
+ loss.backward()
82
+ optimizer.step()
83
+
84
+ #pbar.set_description(f"Loss: {loss.item()}") # Update the progress bar to show the current loss
85
+ w = [param.detach().cpu().numpy() for param in params]
86
+
87
+ return w
88
+
89
+
90
+ def display_sample_pytorch(seed, truncation, directions, distances, scale, start, end, w=None, disp=True, save=None, noise_spec=None):
91
+ # blockPrint()
92
+ model.truncation = truncation
93
+ if w is None:
94
+ w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
95
+ w = [w]*model.get_max_latents() # one per layer
96
+ else:
97
+ w_numpy = [x.cpu().detach().numpy() for x in w]
98
+ w = [np.expand_dims(x, 0) for x in w_numpy]
99
+ #w = [x.unsqueeze(0) for x in w]
100
+
101
+
102
+ for l in range(start, end):
103
+ for i in range(len(directions)):
104
+ w[l] = w[l] + directions[i] * distances[i] * scale
105
+
106
+ w = [torch.from_numpy(x).to(device) for x in w]
107
+ torch.cuda.empty_cache()
108
+ #save image and display
109
+ out = model.sample(w)
110
+ out = out.permute(0, 2, 3, 1).cpu().detach().numpy()
111
+ out = np.clip(out, 0.0, 1.0).squeeze()
112
+
113
+ final_im = Image.fromarray((out * 255).astype(np.uint8)).resize((500,500),Image.LANCZOS)
114
+
115
+
116
+ if save is not None:
117
+ if disp == False:
118
+ print(save)
119
+ final_im.save(f'out/{seed}_{save:05}.png')
120
+ if disp:
121
+ display(final_im)
122
+
123
+ return final_im
124
+
125
+ ## Generate image for app
126
+ def generate_image(truncation, c0, c1, c2, c3, c4, c5, c6, start_layer, end_layer,w):
127
+
128
+ scale = 1
129
+ params = {'c0': c0,
130
+ 'c1': c1,
131
+ 'c2': c2,
132
+ 'c3': c3,
133
+ 'c4': c4,
134
+ 'c5': c5,
135
+ 'c6': c6}
136
+
137
+ param_indexes = {'c0': 0,
138
+ 'c1': 1,
139
+ 'c2': 2,
140
+ 'c3': 3,
141
+ 'c4': 4,
142
+ 'c5': 5,
143
+ 'c6': 6}
144
+
145
+ directions = []
146
+ distances = []
147
+ for k, v in params.items():
148
+ directions.append(latent_dirs[param_indexes[k]])
149
+ distances.append(v)
150
+
151
+ if w is not None:
152
+ w = [torch.from_numpy(x).to(device) for x in w]
153
+
154
+ #w1 = clip_optimized_latent(text1, seed1, iters)
155
+ im = model.sample(w)
156
+ im_np = im.permute(0, 2, 3, 1).cpu().detach().numpy()
157
+ im_np = np.clip(im_np, 0.0, 1.0).squeeze()
158
+
159
+
160
+ input_im = Image.fromarray((im_np * 255).astype(np.uint8))
161
+ seed = 0
162
+
163
+ return input_im, display_sample_pytorch(seed, truncation, directions, distances, scale, int(start_layer), int(end_layer), w=w, disp=False)
164
+
165
+
166
+ # Streamlit app title
167
+ st.image('./pics/logo.jpeg')
168
+ '''## Style One'''
169
+
170
+ @st.cache_resource
171
+ def load_model():
172
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
173
+ # Load the pre-trained CLIP model
174
+ clip_model, clip_preprocess = clip.load("ViT-B/32", device=device)
175
+ inst = get_instrumented_model(config.model, config.output_class,
176
+ config.layer, device, use_w=config.use_w)
177
+ return clip_model, inst
178
+
179
+ # Then, to load your models, call this function:
180
+ clip_model, inst = load_model()
181
+ model = inst.model
182
+
183
+
184
+ path_to_components = get_or_compute(config, inst)
185
+ comps = np.load(path_to_components)
186
+ lst = comps.files
187
+ latent_dirs = []
188
+ latent_stdevs = []
189
+
190
+ load_activations = False
191
+
192
+ for item in lst:
193
+ if load_activations:
194
+ if item == 'act_comp':
195
+ for i in range(comps[item].shape[0]):
196
+ latent_dirs.append(comps[item][i])
197
+ if item == 'act_stdev':
198
+ for i in range(comps[item].shape[0]):
199
+ latent_stdevs.append(comps[item][i])
200
+ else:
201
+ if item == 'lat_comp':
202
+ for i in range(comps[item].shape[0]):
203
+ latent_dirs.append(comps[item][i])
204
+ if item == 'lat_stdev':
205
+ for i in range(comps[item].shape[0]):
206
+ latent_stdevs.append(comps[item][i])
207
+
208
+ ## Side bar texts
209
+ st.sidebar.title('Customization Options')
210
+
211
+
212
+ # Create UI widgets
213
+ text = st.sidebar.text_input("Style Specs", help = "Provide a clear and concise description of the design you wish to generate. This helps the app understand your preferences and create a customized design that matches your vision.")
214
+ if 'seed' not in st.session_state:
215
+ #st.session_state['seed'] = random.randint(1, 1000)
216
+ st.session_state['seed'] = 200
217
+
218
+
219
+ with st.sidebar.expander("Advanced"):
220
+ seed = st.number_input("ID", value= st.session_state['seed'], help = "Capture this unique id to reproduce the exact same result later.")
221
+
222
+ st.session_state['seed'] = seed
223
+ iters = st.number_input("Cycles", value = 25, help = "Increase the sensitivity of the algorithm to find the design matching the style description. Higher values might enhance the accuracy but may lead to slower loading times")
224
+ submit_button = st.sidebar.button("Discover")
225
+ # content = st.sidebar.slider("Structural Composition", min_value=0.0, max_value=1.0, value=0.5)
226
+ # style = st.sidebar.slider("Style", min_value=0.0, max_value=1.0, value=0.5)
227
+ truncation = 0.5
228
+ #truncation = st.sidebar.slider("Dimensional Scaling", min_value=0.0, max_value=1.0, value=0.5)
229
+
230
+ slider_min_val = -20
231
+ slider_max_val = 20
232
+ slider_step = 1
233
+
234
+ c0 = st.sidebar.slider("Sleeve Size Scaling", min_value=slider_min_val, max_value=slider_max_val, value=0, help="Adjust the scaling of sleeve sizes. Increase to make sleeve sizes appear larger, and decrease to make them appear smaller.")
235
+ c1 = st.sidebar.slider("Jacket Features", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Control the prominence of jacket features. Increasing this value will make the features more pronounced, while decreasing it will make them less noticeable")
236
+ c2 = st.sidebar.slider("Women's Overcoat", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Modify the dominance of the women's overcoat style. Increase the value to enhance its prominence, and decrease it to reduce its impact.")
237
+ c3 = st.sidebar.slider("Coat", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Control the prominence of coat features. Increasing this value will make the features more pronounced, while decreasing it will make them less noticeable")
238
+ c4 = st.sidebar.slider("Graphic Elements", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Fine-tune the visibility of graphic elements. Increasing this value will make the graphics more prominent, while decreasing it will make them less visible.")
239
+ c5 = st.sidebar.slider("Darker Color", min_value=slider_min_val, max_value=slider_max_val, value=0, help = "Adjust the intensity of the color tones towards darker shades. Increasing this value will make the colors appear deeper, while decreasing it will lighten the overall color palette.")
240
+ c6 = st.sidebar.slider("Neckline", min_value=slider_min_val, max_value=slider_max_val, value=0,help = "Control the emphasis on the neckline of the garment. Increase to highlight the neckline, and decrease to downplay its prominence.")
241
+ start_layer = 0
242
+ end_layer = 14
243
+ #start_layer = st.sidebar.number_input("Start Layer", value=0)
244
+ #end_layer = st.sidebar.number_input("End Layer", value=14)
245
+
246
+ # if 'w-np' not in st.session_state:
247
+ # st.session_state['w-np'] = None
248
+
249
+ if submit_button: # Execute when the submit button is pressed
250
+ w = clip_optimized_latent(text, seed, iters)
251
+ st.session_state['w-np'] = w
252
+
253
+
254
+ try:
255
+ input_im, output_im = generate_image(truncation, c0, c1, c2, c3, c4, c5, c6, start_layer, end_layer,st.session_state['w-np'])
256
+ st.image(input_im, caption="Input Image")
257
+ st.image(output_im, caption="Output Image")
258
+ except:
259
+ pass
pics/logo.jpeg ADDED
pics/logo_small.jpeg ADDED
requirements.txt CHANGED
@@ -14,4 +14,5 @@ nltk
14
  fbpca
15
  pyopengltk
16
  stqdm
17
- ftfy
 
 
14
  fbpca
15
  pyopengltk
16
  stqdm
17
+ ftfy
18
+ streamlit-extras