import subprocess subprocess.run(['wget', '-N', '-q', 'https://matematica.unipv.it/gualandi/solvers/ipopt-linux64.zip'], stdout=subprocess.DEVNULL) subprocess.run(['unzip', '-o', '-q', 'ipopt-linux64'], stdout=subprocess.DEVNULL) from pyomo.environ import * import pickle import pickle dict_files = { 'banana republic': 'annotations/banana_annotations.pickle', 'h&m': 'annotations/hnm_annotations.pickle', 'uniqlo': 'annotations/uniqlo_annotations.pickle', 'street': 'annotations/street_annotations.pickle', 'abercrombie women': 'annotations/abercrombie_women.pickle', 'aritzia': 'annotations/aritzia.pickle', 'dynamite': 'annotations/dynamite.pickle', } style_gender = { 'banana republic': 'male', 'h&m': 'male', 'uniqlo': 'male', 'street': 'male', 'abercrombie women': 'female', 'aritzia': 'female', 'dynamite': 'female', } gender_article_keys = { 'male': ['shirt', 'pants_or_shorts', 'hoodie_or_jacket', 'hat', 'shoes'], 'female': ['top', 'bottom', 'dress', 'hoodie_or_jacket', 'hat', 'shoes'] } style_annotations_dict = {} def get_annotations(dict_file): with open(dict_file, 'rb') as f: annotations = pickle.load(f) for a in annotations: if not( ( a.get('shirt', '') and a.get('pants_or_shorts', '') ) or ( a.get('top', '') and a.get('bottom', '') ) or a.get('dress', '') ): print( dict_file, '\n', a ) filtered_annotations = [ a for a in annotations if ( (a.get('shirt', '') and a.get('pants_or_shorts', '')) or ( a.get('gender', '') == 'female' ) ) ] return filtered_annotations style_annotations_dict = { k: get_annotations(file_path) for k, file_path in dict_files.items() } import collections from collections import defaultdict optional = [False, False, True, True, False] article_keys = ['shirt', 'pants_or_shorts', 'hoodie_or_jacket', 'hat', 'shoes'] def annotation_to_tuple(annotation): t = tuple( annotation[key] for key in article_keys ) return t def remove_owned_articles(annotations_tuple, clothes_owned): return tuple( article if article not in clothes_owned else '' for article in annotations_tuple ) def annotations_value(annotation_tuple_count, clothes_owned): new_scores = {} for annotation_tuple, count in annotation_tuple_count.items(): num_missing = 0 for clothing in annotation_tuple: if not (clothing == '' or clothing in clothes_owned): num_missing += 1 if num_missing > 0: new_scores[annotation_tuple] = count/num_missing return new_scores def is_outfit_subset(smaller_outfit, larger_outfit): if smaller_outfit == larger_outfit: return False for i in range(len(larger_outfit)): # assume both r tuples of fixed size if smaller_outfit[i] != '' and smaller_outfit[i] != larger_outfit[i]: return False return True def most_outfits_helper(annotations, capacity=5, clothes_owned=[]): annotations = [ annotation_to_tuple(e) for e in annotations ] outfit_count = dict(collections.Counter(annotations)) outfits = list(outfit_count.keys()) for small_outfit in outfit_count: for larger_outfit in outfit_count: if is_outfit_subset(small_outfit, larger_outfit): outfit_count[small_outfit] += outfit_count[larger_outfit] clothes_owned += [''] best_outfits = most_outfits(outfit_count, capacity, set(clothes_owned)) best_outfits.remove('') return best_outfits # consider every image in the dataset # each image has an outfit and casts a vote for that outfit # a outfit is a tuple of pants_or_short + shirt + jacket_or_hoodie? # images with shoes or hats have an additional vote for the shoes and hats # greedily pick the clothes with the highest votes # then remove those clothes from all outfits in that dataset and recount # can be solved as an integer programming problem def most_outfits(annotation_tuple_count, capacity, clothes_owned): if capacity == 0 : return clothes_owned # merge counts based on new keys updated_annotations = defaultdict(int) for tup, count in annotation_tuple_count.items(): updated_annotations[ remove_owned_articles(tup, clothes_owned) ] += count annotation_tuple_count = updated_annotations outfits_scores = annotations_value(annotation_tuple_count, clothes_owned) outfits_scores = sorted(outfits_scores.items(), key=lambda x:-x[1]) num_new = 1 for outfit, score in outfits_scores: clothes_proposed = clothes_owned.union(set(outfit)) num_new = len(clothes_proposed) - len(clothes_owned) if num_new <= capacity: clothes_owned = clothes_proposed break return most_outfits( annotation_tuple_count, capacity-num_new, clothes_owned ) from pyomo.environ import * from functools import reduce def cover_style_ip(annotations, capacity=10, clothes_owned=[], mask=None, solver='ipopt'): """Use integer program to find the set of clothes that makes as many outfits as possible. annotations: List[ Dict ], contains maps from clothing categories to string types capacity: int, number of new clothes to find clothes_owned: List[ Str ], iterable of strings of clothing types mask: Optional( List[ Str ] ), optional iterable of categories of clothes to consider. Uses all clothing types by default. solver: Str, the nonlinear optimization solver to use for max bool sat """ unique_clothes = set() interested_clothing_types = article_keys if mask is None else mask for a in annotations: for key in interested_clothing_types: unique_clothes.add( a[key] ) if '' in unique_clothes: unique_clothes.remove('') model = ConcreteModel() # 1. Create the variables we want to optimize clothing_literals_dict = {clothing: Var(within=Binary) for clothing in unique_clothes} # set literals to true for clothes already owned for clothing in clothes_owned: clothing_literals_dict[clothing] = 1 # capacity changes from number of new clothes to total clothes capacity += len(clothes_owned) capacity_constraint = Constraint(expr=sum(list(clothing_literals_dict.values())) <= capacity) outfit_clauses = [] for a in annotations: # get the clothing literal if it exists, otherwise it's masked and say it's absent outfit_literals = [ clothing_literals_dict.get(a[key], 0) for key in article_keys if a[key] != '' ] outfit_clauses += [ reduce(lambda x,y: x*y, outfit_literals) ] # 3. Maximize the objective function objective = Objective( expr=sum( outfit_clauses ), sense=maximize) for name, literal in clothing_literals_dict.items(): setattr(model, name, literal) model.capacity_constraint = capacity_constraint model.objective = objective SolverFactory(solver).solve(model) basis_clothes = [ name for name, literal in clothing_literals_dict.items() if not isinstance(literal, int) and round(literal()) ] return basis_clothes def annotation_str(annotation): output = "" output += annotation['hat'] output += ' + ' + annotation['shirt'] if output and annotation['shirt'] else annotation['shirt'] output += ' + ' + annotation['hoodie_or_jacket'] if output and annotation['hoodie_or_jacket'] else annotation['hoodie_or_jacket'] output += ' + ' + annotation['pants_or_shorts'] if output and annotation['pants_or_shorts'] else annotation['pants_or_shorts'] output += ' + ' + annotation['shoes'] if output and annotation['shoes'] else annotation['shoes'] # output += ' ' + annotation['url'] if annotation.get('url') else '' return output def annotation_str(annotation): gender = annotation.get('gender', 'male') output = "" for k in gender_article_keys[gender]: output += ' + ' + annotation[k] if output and annotation.get(k) else annotation.get(k, '') return output def annotation_to_key(annotation): gender = annotation.get('gender', 'male') output = "" if gender == 'male': output += ' + ' + annotation['shirt'] if output and annotation['shirt'] else annotation['shirt'] output += ' + ' + annotation['hoodie_or_jacket'] if output and annotation['hoodie_or_jacket'] else annotation['hoodie_or_jacket'] output += ' + ' + annotation['pants_or_shorts'] if output and annotation['pants_or_shorts'] else annotation['pants_or_shorts'] else: useful_keys = ['dress', 'top', 'bottom', 'hoodie_or_jacket'] for k in useful_keys: output += ' + ' + annotation[k] if output and annotation.get(k) else annotation.get(k, '') return output def get_num_outfits(annotations, articles): outfits = set() for e in annotations: if (sum([(e[key] == '' or e[key] in articles ) for key in article_keys]) == len(article_keys) # and e['shirt'] and e['pants_or_shorts'] ): str_form = annotation_to_key(e) # ignore +- hat, shoes otherwise use annotation_str outfits.add(str_form) return sorted(list(outfits)) def get_outfit_urls(annotations, outfits): outfit_urls = defaultdict(list) for e in annotations: str_form = annotation_to_key(e) if e.get('url') and str_form in outfits: outfit_urls[str_form] += [ e['url'] ] return dict(outfit_urls) def cover_style(annotations, capacity=10, clothes_owned=[]): clothes = list(cover_style_ip(annotations, capacity=capacity, clothes_owned=clothes_owned)) reachable_outfits = get_num_outfits(annotations, set(clothes+clothes_owned) ) if len(clothes_owned) == 0: # return basis outfits from scratch return clothes, get_outfit_urls(annotations, reachable_outfits) elif capacity == 0: # return reach of current clothes return clothes, get_outfit_urls(annotations, reachable_outfits) else: # capacity > 0 and len(clothes_owned) > 0, show new clothes and outfits new_clothes = [ c for c in clothes if c not in clothes_owned ] reachable_outfits_base = get_num_outfits(annotations, clothes_owned) new_outfits = [ o for o in reachable_outfits if o not in reachable_outfits_base ] return new_clothes, get_outfit_urls(annotations, new_outfits) def str_to_list(input): tokens = [ token.strip() for token in input.split(',') ] return [ t for t in tokens if t != '' ] CLOTHES_HEADER = f'## Most Effective Clothes' OUTFITS_HEADER = f'## Possible 👕😎 Outfits' def cover_style_helper(styles, capacity=10, clothes_owned=''): if len(styles) == 0: return f'{CLOTHES_HEADER}\nPlease pick at least one style from the left.' , OUTFITS_HEADER clothes_owned = str_to_list(clothes_owned) annotations = [] for style in styles: annotations += style_annotations_dict[style][:500] if len(styles) == 1: # hack for h&m having wayyyyy more examples than other brands annotations = style_annotations_dict[style] clothes, outfit_urls = cover_style(annotations, capacity, clothes_owned) clothes_str = f'{CLOTHES_HEADER}\n' for c in clothes: clothes_str += f'* {c}\n' outfits_str = f'{OUTFITS_HEADER} ({len(outfit_urls)})\n' for outfit, urls in outfit_urls.items(): outfits_str += f'1. {str(outfit)}: ' for i, u in enumerate(urls): outfits_str += f'[{i+1}]' # f'[\[{i}\]]({u})' outfits_str += '\n' return clothes_str, outfits_str # article_keys = gender_article_keys['male'] # print(cover_style_helper( ['banana republic'] )) def cover_style_helper_wrapper(markdown, styles, capacity=10, clothes_owned=''): if len(styles) > 0: global article_keys gender = style_gender[styles[0]] article_keys = gender_article_keys[gender] return cover_style_helper(styles, capacity, clothes_owned) def filter_test(annotation, filter_set): for f in filter_set: num_occur = sum([1 for key in article_keys+['caption'] if f in annotation[key]]) if num_occur == 0: return False return True import gradio as gr def change_gallery(style_choice, start_from=0, text_filter=''): global article_keys gender = style_gender[style_choice] article_keys = gender_article_keys[gender] chosen_annotations = style_annotations_dict[style_choice] if text_filter: text_filter = set([t.strip() for t in text_filter.split(',')]) chosen_annotations = [ a for a in chosen_annotations if filter_test(a, text_filter) ] start_from = int(start_from/100*len(chosen_annotations)) # print(len(chosen_annotations), [a['url'] for a in chosen_annotations[start_from:start_from+20]]) return [a['url'] for a in chosen_annotations[start_from:start_from+20]] def update_styles(gender): global article_keys article_keys = gender_article_keys[gender] default_values = ["banana republic"] if gender == "male" else ["aritzia"] return gr.CheckboxGroup.update(choices=[k for k in style_annotations_dict.keys() if style_gender[k] == gender], value=default_values, label='Styles') article_keys = gender_article_keys['male'] INTRO_MARKDOWN = """**Good clothes are the ones that you can use to make many outfits.** Finding stuff that works with your style and wardrobe is hard. This program helps you find the clothes that make as many fashionable outfits as possible, given your current wardrobe, style, and budget. """ with gr.Blocks() as demo: with gr.Tabs(selected=0): with gr.TabItem('Find the Literal Optimal Clothes', id=0): with gr.Box(): default_clothes = """black dress pants, blue shirt, white sweater, black sweater, white t shirt, black t shirt, black shorts, white shirt, grey hoodie, black t shirt, gray sweater, black jacket, black sneakers, white sneakers, blue jeans, black baseball hat, black jeans""" with gr.Row(): with gr.Column(): gr.Markdown(INTRO_MARKDOWN) with gr.Row(): with gr.Group(): gender = gr.Radio(["male", "female"], value="male", label='gender') styles = gr.CheckboxGroup([k for k in style_annotations_dict.keys() if style_gender[k] == gender.value], value=["banana republic"], label='Styles') gender.change(update_styles, inputs=[gender], outputs=[styles]) with gr.Group(): capacity = gr.Number(5, precision=0, label='Number of clothes to recommend') clothes_owned = gr.Textbox( label="Clothes Owned", lines=3, value=default_clothes, #json.dumps(default_clothes), ) with gr.Row(): btn = gr.Button("Recommend Clothes") with gr.Row(): clothes_markdown, outfits_markdown = cover_style_helper(styles.value, capacity.value, clothes_owned.value) clothes_recommended = gr.Markdown(clothes_markdown) possible_outfits = gr.Markdown(outfits_markdown) btn.click(cover_style_helper, inputs=[styles, capacity, clothes_owned], outputs=[clothes_recommended, possible_outfits]) gr.Markdown("## 3 Different Example Uses") example_label = gr.Textbox('', label='Explanation', visible=False) gr.Examples(examples=[ ['Find the central clothes that make a style.', ['street'], 10, ''], ['Find new outfits hidden in your wardrobe.', ['h&m'], 0, 'white t shirt, grey t shirt, black pants, black shorts, black t shirt, blue jeans'], ['Find the clothes that best fit your existing wardrobe.', ['banana republic', 'street'], 5, 'white t shirt, grey t shirt, black pants, black shorts, black t shirt, blue jeans']], inputs=[example_label, styles, capacity, clothes_owned], outputs=[clothes_recommended, possible_outfits], fn=cover_style_helper_wrapper, cache_examples=False) EXPLORE_MARKDOWN = """# Explore Styles Hint: Click an image to enable arrow key browsing. You put in clothes you own to find outfit ideas! """ with gr.TabItem('Explore Styles', id=1): with gr.Box(): gr.Markdown(EXPLORE_MARKDOWN) with gr.Row(): with gr.Column(): bad_urls = ['dynamite', 'abercrombie women'] styles = gr.Radio([ k for k in style_annotations_dict.keys() if k not in bad_urls ], value="uniqlo", label='Styles') start_from = gr.Slider(0, 100, label='Start from', value=0, step=10) with gr.Group(): text_filter = gr.Textbox(value='white t shirt, jeans', label='Outfit includes') filter_button = gr.Button('Apply Text Filter') with gr.Column(): outfits_gallery = gr.Gallery(label='Outfit Examples', value=change_gallery(styles.value, start_from.value, text_filter.value)) styles.change(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery) start_from.change(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery) filter_button.click(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery) with gr.TabItem('How it works', id=2): ABOUT_MARKDOWN = """ # Why I don't really enjoy shopping, it takes a while a look at all the clothes, and I'm not sure if I should buy what I picked out. This program can tell me which clothes are the best for my wardrobe and style, so I don't have to worry about buying the wrong stuff. # How This program poses the problem as a nonlinear integer programming problem. The problem is to maximize the number of outfits, while contraining the number of clothes to be <= "this max you want to buy". Let a, b, c etc be binary literals that represent the use of clothing type a, b, c. These are clothing types like white dress shirt, or black hat. We maximize the nonlinear expression abc + ade + bec + ... similar to a disjuctive normal form, where each clause represents an outfit and is the product of clothing literals. Each outfit clause has value 1 iff every article of clothing is used in the solution. Pyomo is used to solve this optimization problem. This objective takes the form of something like, ![objective problem](https://github.com/Ryan-Qiyu-Jiang/writting/blob/master/clothing_optimization_problem.png?raw=true) Hart, William E., Carl Laird, Jean-Paul Watson, David L. Woodruff, Gabriel A. Hackebeil, Bethany L. Nicholson, and John D. Siirola. Pyomo – Optimization Modeling in Python. Springer, 2017. # Caveats This model understands fashion from a macro level. It understands what a white shirt is, but is entirely blind to micro features like fit, brand, or shape. It also bins different kinds of clothing and colors together. So the model could mix up light grey sweatpants and dark grey chinos because they're both grey pants. There is also error introduced from bad image annotations, approximate solvers, and style dataset size. Hence it's important to look at the outfits images under possible outfits for a human touch! """ with gr.Box(): gr.Markdown(ABOUT_MARKDOWN) article_keys = gender_article_keys['male'] demo.launch(debug=False)