fashion

README.md

@@ -1,13 +1,12 @@
 ---
 title: ModelProblems
-emoji: 🧠
+emoji: ⚖️ 
 colorFrom: pink
-colorTo: indigo
+colorTo: green
 sdk: gradio
 sdk_version: 4.28.3
 app_file: app.py
-pinned: false
+pinned: true
 license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -13,25 +13,46 @@ from PIL import Image
 from huggingface_hub import from_pretrained_keras
 from math import sqrt, ceil
 import numpy as np
+from transformers import pipeline
 import pandas as pd
 import gradio as gr
+import base64
 
 modelieo=[
  'nathanReitinger/MNIST-diffusion',
  'nathanReitinger/MNIST-diffusion-oneImage',
  'nathanReitinger/MNIST-GAN',
- 'nathanReitinger/MNIST-GAN-noDropout'
+ 'nathanReitinger/MNIST-GAN-noDropout',
+ 'nathanReitinger/FASHION-diffusion-oneImage'
   ]
 
-def get_sims(gen_filepath, gen_label, file_path, hunting_time_limit):
-  (train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()
-  train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')
-  train_images = (train_images - 127.5) / 127.5	# Normalize the images to [-1, 1]
+def get_sims(gen_filepath, gen_label, file_path, hunting_time_limit, data_type):
+  if data_type == 'mnist':
+    (train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()
+    train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')
+    train_images = (train_images - 127.5) / 127.5	# Normalize the images to [-1, 1]
+  else:
+    (train_images, train_labels), (_, _) = tf.keras.datasets.fashion_mnist.load_data()
+    train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')
+    train_images = (train_images - 127.5) / 127.5	# Normalize the images to [-1, 1]
 
   print("how long to hunt", hunting_time_limit)
   if hunting_time_limit == None:
     hunting_time_limit = 2
 
+  train_label_mapping = {
+    0: 'T - shirt / top',
+    1: 'Trouser',
+    2: 'Pullover',
+    3: 'Dress',
+    4: 'Coat',
+    5: 'Sandal',
+    6: 'Shirt',
+    7: 'Sneaker',
+    8: 'Bag',
+    9: 'Ankle boot'
+  }
+
   lowest_score = 10000
   lowest_image = None
   lowest_image_path = ''
@@ -39,8 +60,14 @@ def get_sims(gen_filepath, gen_label, file_path, hunting_time_limit):
   start = time.time()
 
   for i in range(len(train_labels)):
-    # print(i)
-    if train_labels[i] == gen_label:
+    if data_type == 'fashion':
+      label_option = train_label_mapping[train_labels[i]]
+    else:
+      label_option = train_labels[i]
+    # print(data_type, i, label_option, gen_label)
+    if label_option == gen_label:
+
+        print('match on types!')
 
         ###
         # get a real image (of correct number)
@@ -84,38 +111,54 @@ def get_sims(gen_filepath, gen_label, file_path, hunting_time_limit):
   return [lowest_image_path, lowest_score]
 
 
-def digit_recognition(filename):
-
-  API_URL = "https://api-inference.huggingface.co/models/farleyknight/mnist-digit-classification-2022-09-04"
-  special_string = '-h-f-_-RT-U-J-E-M-Pb-GC-c-i-v-sji-bMsQmxuh-x-h-C-W-B-F-W-z-Gv-'
-  is_escaped = special_string.replace("-", '')
-  bear = "Bearer " + is_escaped
-  headers = {"Authorization": bear}
-  # get a prediction on what number this is
-  def query(filename):
-    with open(filename, "rb") as f:
-      data = f.read()
-    response = requests.post(API_URL, headers=headers, data=data)
-    return response.json()
-
-  # use latest model to generate a new image, return path
-  ret = False 
-  output = None
-  while ret == False:
-    output = query(filename + 'generated_image.png')
-    if 'error' in output:
-      time.sleep(10)
-      ret = False 
-    else:
-      ret = True
-  print(output)
+def digit_recognition(filename, data_type):
+
+  if data_type == 'mnist':
+    # API_URL = "https://api-inference.huggingface.co/models/farleyknight/mnist-digit-classification-2022-09-04"
+    # special_string = '-h-f-_-RT-U-J-E-M-Pb-GC-c-i-v-sji-bMsQmxuh-x-h-C-W-B-F-W-z-Gv-'
+    # is_escaped = special_string.replace("-", '')
+    # bear = "Bearer " + is_escaped
+    # headers = {"Authorization": bear}
+    # # get a prediction on what number this is
+    # def query(filename):
+    #   with open(filename, "rb") as f:
+    #     data = f.read()
+    #   response = requests.post(API_URL, headers=headers, data=data)
+    #   return response.json()
+
+    # # use latest model to generate a new image, return path
+    # ret = False 
+    # output = None
+    # while ret == False:
+    #   output = query(filename + 'generated_image.png')
+    #   if 'error' in output:
+    #     time.sleep(10)
+    #     ret = False 
+    #   else:
+    #     ret = True
+
+    # slower than inferenceAPI, but no tokens needed
+    pipe = pipeline("image-classification", model="farleyknight/mnist-digit-classification-2022-09-04")
+    output = pipe(filename + 'generated_image.png')
+    print(output)
+    this_label_for_this_image = int(output[0]['label'])
+  
+  else:
+    
+
+    pipe = pipeline("image-classification", model="nathanReitinger/FASHION-vision")
+    output = pipe(filename + 'generated_image.png')
+    this_label_for_this_image = output[0]['label']
+    print(output)
+    print(this_label_for_this_image)
+
+
+  print(output, this_label_for_this_image)
 
-  low_score_log = ''
-  this_label_for_this_image = int(output[0]['label'])
   return {'full': output, 'number': this_label_for_this_image}
 
 
-def get_other(original_image, hunting_time_limit):
+def get_other(original_image, hunting_time_limit, data_type):
   RANDO = str(time.time())
   file_path = 'tester/' + 'generation' + "/" + RANDO + '/'
   os.makedirs(file_path)
@@ -127,17 +170,21 @@ def get_other(original_image, hunting_time_limit):
   plt.close()
   print('[+] done saving generation')
   print("[-] what digit is this")
-  ret = digit_recognition(file_path)
+  print(data_type)
+  # sys.exit()
+  ret = digit_recognition(file_path, data_type)
   print(ret['full'])
   print(ret['number'])
   print("[+]", ret['number'])
   print("[-] show some most similar numbers")
   if ret["full"][0]['score'] <= 0.90:
-    print("[!] error in image digit recognition, likely to not find a similar score")
-    sys.exit()
+    print("[!] error in image recognition, likely to not find a similar score")
+    return (file_path + 'generated_image.png', ['error.png', -1])
+    # sys.exit()
   gen_filepath = file_path + 'generated_image.png'
   gen_label = ret['number']
-  ret_sims = get_sims(gen_filepath, gen_label, file_path, hunting_time_limit)
+  ret_sims = get_sims(gen_filepath, gen_label, file_path, hunting_time_limit, data_type)
+  print(ret_sims)
   print("[+] done sims")
   # get the file-Path
   return (file_path + 'generated_image.png', ret_sims)
@@ -169,25 +216,29 @@ def generate_and_save_images(model):
 def TextToImage(Prompt,inference_steps, model):
   model_id = model
   if 'GAN' in model_id:
-    print("do something else")
+    print("--> GAN <--")
     model = from_pretrained_keras(model)
     image = generate_and_save_images(model)
+  
   else:
+    print("--> DIFFUSION <--")
     pipe = DiffusionPipeline.from_pretrained(model_id)
     the_randomness = int(str(time.time())[-1])
     print('seed', the_randomness)
     image = pipe(generator= torch.manual_seed(the_randomness), num_inference_steps=inference_steps).images[0]
 
-#   pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
-#   pipe = pipe.to("cpu")
-
   prompt = Prompt
   print(prompt)
   hunting_time_limit = None
   if prompt.isnumeric():
     hunting_time_limit = abs(int(prompt))
+  
+  if 'FASHION' in model_id:
+    data_type = 'fashion'
+  if 'MNIST' in model_id:
+    data_type = 'mnist'
 
-  original_image, other_images = get_other(image, hunting_time_limit)
+  original_image, other_images = get_other(image, hunting_time_limit, data_type=data_type)
   the_file = other_images[0]
   the_rmse = other_images[1]
   ai_gen = Image.open(open(original_image, 'rb'))
@@ -198,10 +249,10 @@ def TextToImage(Prompt,inference_steps, model):
   return [ai_gen, another_one]
 
 df = pd.DataFrame({
-    "Model"                 : ['MNIST-diffusion', 'MNIST-diffusion-oneImage', 'MNIST-GAN', 'MNIST-GAN-noDropout'], 
-    "Class (Architecture)"  : ['UNet2DModel', 'UNet2DModel', 'Sequential', 'Sequential'], 
-    "Dataset Examples"      : [60000, 1, 60000, 60000], 
-    "Notes"                 : ['Similar architecture as Stable Diffusion, different training data', 'Toy model, purposed to store protected content', 'GANs are not as likely to store protected content', 'less dropout, more copying?']
+    "Model"                 : ['MNIST-diffusion', 'MNIST-diffusion-oneImage', 'MNIST-GAN', 'MNIST-GAN-noDropout', 'FASHION-diffuion-oneImage'], 
+    "Class (Architecture)"  : ['UNet2DModel', 'UNet2DModel', 'Sequential', 'Sequential', 'UNet2DModel'], 
+    "Dataset Examples"      : [60000, 1, 60000, 60000, 1], 
+    "Notes"                 : ['Similar architecture as Stable Diffusion, different training data', 'Toy model, purposed to store protected content', 'GANs are not as likely to store protected content', 'less dropout, more copying?', 'same diffusion, different data (more variance in data)']
 }) 
 
 # Applying style to highlight the maximum value in each row
@@ -219,7 +270,7 @@ with gr.Blocks() as app:
       "<hr>"
       "<h1><center>Do machine learing models store protected content?</center></h1>" +
       "<p><center><span style='color: red;'>Enter a time to hunt for copies (seconds), select a model, and hit submit!</center></p>" +
-      "<p><center><strong>These image generation models will give you a 'bespoke' generation ❤ of an <a href='https://paperswithcode.com/dataset/mnist'>MNIST hand-drawn digit</a></p> " +
+      "<p><center><strong>These image generation models will give you a 'bespoke' generation ❤ of an <a href='https://paperswithcode.com/dataset/mnist'>MNIST hand-drawn digit</a> or the <a href='https://www.tensorflow.org/datasets/catalog/fashion_mnist'>fashion dataset</a></p>  " +
       "<p><center>then the program will search in training data (for <i>n</i> seconds) to find similar images: <a href='https://medium.com/@mygreatlearning/rmse-what-does-it-mean-2d446c0b1d0e'>RMSE</a>, lower is more similar</p>" +
       "<p><a href='https://nathanreitinger.umiacs.io'>@nathanReitinger</a></p>"
   )