Initial Commit

annotations/abercrombie_women.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2489e4154b2d035c7c0ddee66100e642da754c03e005e292a301618abe70cf06
+size 67188

annotations/aritzia.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9501466b2d4327aa29a2fdfcdb2de712821323dba16cbb7c200d5107ecf5814
+size 90000

annotations/banana_annotations.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61b679f59f989e012530ab5c3c2bf5246507b34c9aa4844ffe9e93d552e3fb71
+size 49980

annotations/dynamite.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d9babf9f122ebae53223e2ac8edb6c2666a304df614239876bab91c3b8e93fe
+size 64847

annotations/hnm_annotations.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5eb93941f94524318a41537e90c2a2df1ffdf094c19cb3f9b06f3e5bf7af18b
+size 444989

annotations/street_annotations.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:573e43757b4348138156518f25dffed3260659382be2aa5c5ff47db56b7c18a4
+size 80297

annotations/uniqlo_annotations.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64a78ebb310af49d972e9b579b42fac8b8042ea5c06669610a8a542435cf649d
+size 38733

app.py

@@ -0,0 +1,452 @@
+import subprocess
+subprocess.run(['wget', '-N', '-q', 'https://ampl.com/dl/open/ipopt/ipopt-linux64.zip'], stdout=subprocess.DEVNULL)
+subprocess.run(['unzip', '-o', '-q', 'ipopt-linux64'], stdout=subprocess.DEVNULL)
+
+from pyomo.environ import *
+import pickle
+
+import pickle
+dict_files = {
+    'banana republic': 'annotations/banana_annotations.pickle',
+    'h&m': 'annotations/hnm_annotations.pickle',
+    'uniqlo': 'annotations/uniqlo_annotations.pickle',
+    'street': 'annotations/street_annotations.pickle',
+    'abercrombie women': 'annotations/abercrombie_women.pickle',
+    'aritzia': 'annotations/aritzia.pickle',
+    'dynamite': 'annotations/dynamite.pickle',
+}
+
+style_gender = {
+    'banana republic': 'male',
+    'h&m': 'male',
+    'uniqlo': 'male',
+    'street': 'male',
+    'abercrombie women': 'female',
+    'aritzia': 'female',
+    'dynamite': 'female',
+}
+
+gender_article_keys = {
+    'male': ['shirt', 'pants_or_shorts', 'hoodie_or_jacket', 'hat', 'shoes'],
+    'female': ['top', 'bottom', 'dress', 'hoodie_or_jacket', 'hat', 'shoes']
+}
+
+style_annotations_dict = {}
+
+def get_annotations(dict_file):
+  with open(dict_file, 'rb') as f:
+      annotations = pickle.load(f)
+
+  for a in annotations:
+    if not( ( a.get('shirt', '') and a.get('pants_or_shorts', '') ) or 
+            ( a.get('top', '') and a.get('bottom', '') ) or
+              a.get('dress', '') 
+           ):
+      print( dict_file, '\n', a )
+
+  filtered_annotations = [ a for a in annotations if ( (a.get('shirt', '') and a.get('pants_or_shorts', '')) or (
+                                                                                          a.get('gender', '') == 'female'  ) ) ]
+  return filtered_annotations
+
+style_annotations_dict = {
+    k: get_annotations(file_path) for k, file_path in dict_files.items()
+}
+
+import collections
+from collections import defaultdict
+
+optional = [False, False, True, True, False]
+article_keys = ['shirt', 'pants_or_shorts', 'hoodie_or_jacket', 'hat', 'shoes']
+
+def annotation_to_tuple(annotation):
+  t = tuple( annotation[key] for key in article_keys )
+  return t
+
+def remove_owned_articles(annotations_tuple, clothes_owned):
+  return tuple( article if article not in clothes_owned else '' 
+               for article in annotations_tuple )
+
+def annotations_value(annotation_tuple_count, clothes_owned):
+  new_scores = {}
+  for annotation_tuple, count in annotation_tuple_count.items():
+    num_missing = 0
+    for clothing in annotation_tuple:
+
+      if not (clothing == '' or clothing in clothes_owned):
+        num_missing += 1
+    if num_missing > 0:
+      new_scores[annotation_tuple] = count/num_missing
+  return new_scores
+
+def is_outfit_subset(smaller_outfit, larger_outfit):
+  if smaller_outfit == larger_outfit:
+    return False
+  for i in range(len(larger_outfit)): # assume both r tuples of fixed size
+    if smaller_outfit[i] != '' and smaller_outfit[i] != larger_outfit[i]:
+      return False
+  return True
+
+def most_outfits_helper(annotations, capacity=5, clothes_owned=[]):
+  annotations = [ annotation_to_tuple(e) for e in annotations ]
+  outfit_count = dict(collections.Counter(annotations))
+
+  outfits = list(outfit_count.keys())
+  for small_outfit in outfit_count:
+    for larger_outfit in outfit_count:
+      if is_outfit_subset(small_outfit, larger_outfit):
+        outfit_count[small_outfit] += outfit_count[larger_outfit]
+
+  clothes_owned += ['']
+  best_outfits = most_outfits(outfit_count, capacity, set(clothes_owned))
+  best_outfits.remove('')
+  return best_outfits
+
+# consider every image in the dataset
+# each image has an outfit and casts a vote for that outfit
+# a outfit is a tuple of pants_or_short + shirt + jacket_or_hoodie?
+# images with shoes or hats have an additional vote for the shoes and hats
+# greedily pick the clothes with the highest votes
+# then remove those clothes from all outfits in that dataset and recount
+# can be solved as an integer programming problem
+def most_outfits(annotation_tuple_count, capacity, clothes_owned):
+  if capacity == 0 :
+    return clothes_owned
+
+  # merge counts based on new keys
+  updated_annotations = defaultdict(int)
+  for tup, count in annotation_tuple_count.items():
+    updated_annotations[ remove_owned_articles(tup, clothes_owned) ] += count
+  annotation_tuple_count = updated_annotations
+
+  outfits_scores = annotations_value(annotation_tuple_count, clothes_owned)
+  outfits_scores = sorted(outfits_scores.items(), key=lambda x:-x[1])
+  num_new = 1
+  for outfit, score in outfits_scores:
+    clothes_proposed = clothes_owned.union(set(outfit))
+    num_new = len(clothes_proposed) - len(clothes_owned)
+    if num_new <= capacity:
+      clothes_owned = clothes_proposed
+      break
+
+  return most_outfits( annotation_tuple_count, capacity-num_new, clothes_owned )
+
+from pyomo.environ import *
+from functools import reduce
+
+def cover_style_ip(annotations, capacity=10, clothes_owned=[], mask=None, solver='ipopt'):
+    """Use integer program to find the set of clothes that makes as many outfits as possible.
+      annotations: List[ Dict ], contains maps from clothing categories to string types
+      capacity: int, number of new clothes to find
+      clothes_owned: List[ Str ], iterable of strings of clothing types
+      mask: Optional( List[ Str ] ), optional iterable of categories of clothes to consider.
+        Uses all clothing types by default.
+      solver: Str, the nonlinear optimization solver to use for max bool sat
+    """
+    unique_clothes = set()
+    interested_clothing_types = article_keys if mask is None else mask
+    for a in annotations:
+      for key in interested_clothing_types:
+        unique_clothes.add( a[key] )
+    if '' in unique_clothes:
+      unique_clothes.remove('')
+
+    model = ConcreteModel()
+
+    # 1. Create the variables we want to optimize
+    clothing_literals_dict = {clothing: Var(within=Binary) for clothing in unique_clothes}
+
+    # set literals to true for clothes already owned
+    for clothing in clothes_owned:
+      clothing_literals_dict[clothing] = 1
+
+    # capacity changes from number of new clothes to total clothes
+    capacity += len(clothes_owned)
+
+    capacity_constraint = Constraint(expr=sum(list(clothing_literals_dict.values())) <= capacity)
+
+    outfit_clauses = []
+    for a in annotations:
+      # get the clothing literal if it exists, otherwise it's masked and say it's absent
+      outfit_literals = [ clothing_literals_dict.get(a[key], 0) for key in article_keys if a[key] != '' ]
+      outfit_clauses += [ reduce(lambda x,y: x*y, outfit_literals) ]
+
+    # 3. Maximize the objective function
+    objective = Objective( expr=sum( outfit_clauses ), sense=maximize)
+
+    for name, literal in clothing_literals_dict.items():
+      setattr(model, name, literal)
+    model.capacity_constraint = capacity_constraint
+    model.objective = objective
+
+    SolverFactory(solver).solve(model)
+
+    basis_clothes = [ name for name, literal in clothing_literals_dict.items() if not isinstance(literal, int) and round(literal()) ]
+    return basis_clothes
+
+def annotation_str(annotation):
+  output = ""
+  output += annotation['hat']
+  output += ' + ' + annotation['shirt'] if output and annotation['shirt'] else annotation['shirt']
+  output += ' + ' + annotation['hoodie_or_jacket'] if output and annotation['hoodie_or_jacket'] else annotation['hoodie_or_jacket']
+  output += ' + ' + annotation['pants_or_shorts'] if output and annotation['pants_or_shorts'] else annotation['pants_or_shorts']
+  output += ' + ' + annotation['shoes'] if output and annotation['shoes'] else annotation['shoes']
+  # output += ' ' + annotation['url'] if annotation.get('url') else ''
+  return output
+
+def annotation_str(annotation):
+  gender = annotation.get('gender', 'male')
+  output = ""
+  for k in gender_article_keys[gender]:
+    output += ' + ' + annotation[k] if output and annotation.get(k) else annotation.get(k, '')
+  return output
+  
+
+def annotation_to_key(annotation):
+  gender = annotation.get('gender', 'male')
+  output = ""
+  if gender == 'male':
+    output += ' + ' + annotation['shirt'] if output and annotation['shirt'] else annotation['shirt']
+    output += ' + ' + annotation['hoodie_or_jacket'] if output and annotation['hoodie_or_jacket'] else annotation['hoodie_or_jacket']
+    output += ' + ' + annotation['pants_or_shorts'] if output and annotation['pants_or_shorts'] else annotation['pants_or_shorts']
+  else:
+    useful_keys = ['dress', 'top', 'bottom', 'hoodie_or_jacket']
+    for k in useful_keys:
+      output += ' + ' + annotation[k] if output and annotation.get(k) else annotation.get(k, '')
+  return output
+
+
+def get_num_outfits(annotations, articles):
+  outfits = set()
+  for e in annotations:
+    if (sum([(e[key] == '' or e[key] in articles ) for key in article_keys]) == len(article_keys)
+      # and e['shirt'] and e['pants_or_shorts']
+      ):
+      str_form = annotation_to_key(e) # ignore +- hat, shoes otherwise use annotation_str
+      outfits.add(str_form)
+  return sorted(list(outfits))
+
+def get_outfit_urls(annotations, outfits):
+  outfit_urls = defaultdict(list)
+
+  for e in annotations:
+    str_form = annotation_to_key(e)
+    if e.get('url') and str_form in outfits:
+      outfit_urls[str_form] += [ e['url'] ]
+  return dict(outfit_urls)
+
+def cover_style(annotations, capacity=10, clothes_owned=[]):
+  clothes = list(cover_style_ip(annotations, capacity=capacity, clothes_owned=clothes_owned))
+  reachable_outfits = get_num_outfits(annotations, set(clothes+clothes_owned) )
+
+  if len(clothes_owned) == 0: # return basis outfits from scratch
+    return clothes, get_outfit_urls(annotations, reachable_outfits)
+
+  elif capacity == 0: # return reach of current clothes
+    return clothes, get_outfit_urls(annotations, reachable_outfits)
+
+  else: # capacity > 0 and len(clothes_owned) > 0, show new clothes and outfits
+    new_clothes = [ c for c in clothes if c not in clothes_owned ]
+    reachable_outfits_base = get_num_outfits(annotations, clothes_owned)
+    new_outfits = [ o for o in reachable_outfits if o not in reachable_outfits_base ]
+    return new_clothes, get_outfit_urls(annotations, new_outfits)
+
+
+def str_to_list(input):
+  tokens = [ token.strip() for token in input.split(',') ]
+  return [ t for t in tokens if t != '' ]
+
+CLOTHES_HEADER = f'## Most Effective Clothes'
+OUTFITS_HEADER = f'## Possible 👕😎 Outfits'
+
+def cover_style_helper(styles, capacity=10, clothes_owned=''):
+  
+  if len(styles) == 0:
+    return f'{CLOTHES_HEADER}\nPlease pick at least one style from the left.' , OUTFITS_HEADER
+
+  clothes_owned = str_to_list(clothes_owned)
+  annotations = []
+  for style in styles:
+    annotations += style_annotations_dict[style][:500]
+
+  if len(styles) == 1: # hack for h&m having wayyyyy more examples than other brands
+    annotations = style_annotations_dict[style]
+
+  clothes, outfit_urls = cover_style(annotations, capacity, clothes_owned)
+  clothes_str = f'{CLOTHES_HEADER}\n'
+  for c in clothes:
+    clothes_str += f'* {c}\n'
+
+  outfits_str = f'{OUTFITS_HEADER} ({len(outfit_urls)})\n'
+
+  for outfit, urls in outfit_urls.items():
+    outfits_str += f'1. {str(outfit)}: '
+    for i, u in enumerate(urls):
+      outfits_str += f'<a href="{u}" target="_blank">[{i+1}]</a>' # f'[\[{i}\]]({u})'
+    outfits_str += '\n'
+  return clothes_str, outfits_str
+
+article_keys = gender_article_keys['male']
+print(cover_style_helper( ['banana republic'] ))
+
+def cover_style_helper_wrapper(markdown, styles, capacity=10, clothes_owned=''):
+  if len(styles) > 0:
+    global article_keys
+    gender = style_gender[styles[0]]
+    article_keys = gender_article_keys[gender]
+  return cover_style_helper(styles, capacity, clothes_owned)
+
+def filter_test(annotation, filter_set):
+  for f in filter_set:
+    num_occur = sum([1 for key in article_keys+['caption'] if f in annotation[key]])
+    if num_occur == 0:
+      return False
+  return True
+
+import gradio as gr
+
+def change_gallery(style_choice, start_from=0, text_filter=''):
+  global article_keys
+  gender = style_gender[style_choice]
+  article_keys = gender_article_keys[gender]
+  
+  chosen_annotations = style_annotations_dict[style_choice]
+  if text_filter:
+    text_filter = set([t.strip() for t in text_filter.split(',')])
+    chosen_annotations = [ a for a in chosen_annotations if filter_test(a, text_filter) ]
+  start_from = int(start_from/100*len(chosen_annotations))
+  # print(len(chosen_annotations), [a['url'] for a in chosen_annotations[start_from:start_from+20]])
+  return [a['url'] for a in chosen_annotations[start_from:start_from+20]]  
+
+def update_styles(gender):
+  global article_keys
+  article_keys = gender_article_keys[gender]
+
+  default_values = ["banana republic"] if gender == "male" else ["aritzia"]
+  return gr.CheckboxGroup.update(choices=[k for k in style_annotations_dict.keys() if style_gender[k] == gender], value=default_values, label='Styles')
+
+
+
+
+
+article_keys = gender_article_keys['male']
+INTRO_MARKDOWN = """**Good clothes are the ones that you can use to make many outfits.**  
+Finding stuff that works with your style and wardrobe is hard.  
+This program helps you find the clothes that make as many fashionable outfits as possible,  
+given your current wardrobe, style, and budget.  
+
+"""
+
+with gr.Blocks() as demo:
+  with gr.Tabs(selected=0):
+    with gr.TabItem('Find the Literal Optimal Clothes', id=0):
+      with gr.Box():
+        default_clothes = """black dress pants, 
+blue dress shirt, 
+blue jacket, 
+blue jeans, 
+blue shirt, 
+brown boots, 
+gray sweater, 
+white dress shirt, 
+white sneakers"""
+
+        with gr.Row():
+          with gr.Column():
+            gr.Markdown(INTRO_MARKDOWN)
+            with gr.Row():
+              with gr.Group():
+                gender = gr.Radio(["male", "female"], value="male", label='gender')
+                styles = gr.CheckboxGroup([k for k in style_annotations_dict.keys() if style_gender[k] == gender.value], value=["banana republic"], label='Styles')
+                gender.change(update_styles, inputs=[gender], outputs=[styles])
+              with gr.Group():
+                capacity = gr.Number(5, precision=0, label='Number of clothes to recommend')
+                clothes_owned = gr.Textbox(
+                        label="Clothes Owned",
+                        lines=3,
+                        value=default_clothes, #json.dumps(default_clothes),
+                )
+            with gr.Row():
+              btn = gr.Button("Recommend Clothes")
+
+          with gr.Row():
+              clothes_markdown, outfits_markdown = cover_style_helper(styles.value, capacity.value, clothes_owned.value)
+              clothes_recommended = gr.Markdown(clothes_markdown)
+              possible_outfits = gr.Markdown(outfits_markdown)
+              
+          btn.click(cover_style_helper, inputs=[styles, capacity, clothes_owned], outputs=[clothes_recommended, possible_outfits])
+
+        gr.Markdown("## 3 Different Example Uses")
+        example_label = gr.Textbox('', label='Explanation', visible=False)
+        gr.Examples(examples=[ ['Find the central clothes that make a style.', ['street'], 10, ''],
+                              ['Find new outfits hidden in your wardrobe.', ['h&m'], 0, 'white t shirt, grey t shirt, black pants, black shorts, black t shirt, blue jeans'],
+                              ['Find the clothes that best fit your existing wardrobe.', ['banana republic', 'street'], 5, 'white t shirt, grey t shirt, black pants, black shorts, black t shirt, blue jeans']], 
+                    inputs=[example_label, styles, capacity, clothes_owned], 
+                    outputs=[clothes_recommended, possible_outfits], 
+                    fn=cover_style_helper_wrapper, 
+                    cache_examples=False)
+
+    EXPLORE_MARKDOWN = """# Explore Styles
+Hint: Click an image to enable arrow key browsing.  
+You put in clothes you own to find outfit ideas!
+"""
+    with gr.TabItem('Explore Styles', id=1):
+      with gr.Box():
+        gr.Markdown(EXPLORE_MARKDOWN)
+        with gr.Row():
+          with gr.Column():
+            bad_urls = ['dynamite', 'abercrombie women']
+            styles = gr.Radio(list(style_annotations_dict.keys()), value="uniqlo", label='Styles')
+            start_from = gr.Slider(0, 100, label='Start from', value=0, step=10)
+            with gr.Group():
+              text_filter = gr.Textbox(value='white t shirt, jeans', label='Outfit includes')
+              filter_button = gr.Button('Apply Text Filter')
+          with gr.Column():
+            outfits_gallery = gr.Gallery(label='Outfit Examples', value=change_gallery(styles.value, start_from.value, text_filter.value))
+
+        styles.change(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery)
+        start_from.change(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery)
+        filter_button.click(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery)
+
+    with gr.TabItem('How it works', id=2):
+      ABOUT_MARKDOWN = """
+# Why
+I don't really enjoy shopping, it takes a while a look at all the clothes,  
+and I'm not sure if I should buy what I picked out.  
+This program can tell me which clothes are the best for my wardrobe and style,  
+so I don't have to worry about buying the wrong stuff.  
+
+# How
+This program poses the problem as a nonlinear integer programming problem.  
+The problem is to maximize the number of outfits, while contraining the number of clothes to be <= "this max you want to buy".  
+Let a, b, c etc be binary literals that represent the use of clothing type a, b, c.  
+These are clothing types like white dress shirt, or black hat.  
+We maximize the nonlinear expression abc + ade + bec + ... similar to a disjuctive normal form,
+where each clause represents an outfit and is the product of clothing literals.  
+Each outfit clause has value 1 iff every article of clothing is used in the solution.  
+Pyomo is used to solve this optimization problem.  
+
+This objective takes the form of something like,  
+<iframe src="https://drive.google.com/file/d/1j0bBRZkdc7Ce5KWawlwrV36lI0MsMLRf/preview" width="640" height="480" allow="autoplay"></iframe>
+
+Hart, William E., Carl Laird, Jean-Paul Watson, David L. Woodruff, Gabriel A. Hackebeil, Bethany L. Nicholson, and John D. Siirola. Pyomo – Optimization Modeling in Python. Springer, 2017.
+
+
+# Caveats
+This model understands fashion from a macro level.
+It understands what a white shirt is, but is entirely blind to micro features like fit, brand, or shape.  
+It also bins different kinds of clothing and colors together.
+So the model could mix up light grey sweatpants and dark grey chinos because they're both grey pants.  
+
+There is also error introduced from bad image annotations, approximate solvers, and style dataset size.  
+
+Hence it's important to look at the outfits images under possible outfits for a human touch!
+"""
+      with gr.Box():
+        gr.Markdown(ABOUT_MARKDOWN)
+
+
+
+demo.launch(debug=True)
+
+
+

requirements.txt

@@ -0,0 +1,2 @@
+gradio
+pyomo