sohojoe commited on
Commit
b2b5d5f
1 Parent(s): 0441b41

experiments with open_clip, templates, clustering, recursion

Browse files
experimental/clip_app.py CHANGED
@@ -11,6 +11,7 @@ from clip_retrieval.load_clip import load_clip, get_tokenizer
11
  # from clip_retrieval.clip_client import ClipClient, Modality
12
 
13
  @serve.deployment(num_replicas=6, ray_actor_options={"num_cpus": .2, "num_gpus": 0.1})
 
14
  class CLIPTransform:
15
  def __init__(self):
16
  # os.environ["OMP_NUM_THREADS"] = "20"
@@ -18,7 +19,7 @@ class CLIPTransform:
18
  # Load model
19
  self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
20
  self._clip_model="ViT-L/14"
21
- self._clip_model_id ="laion5B-L-14"
22
  self.model, self.preprocess = load_clip(self._clip_model, use_jit=True, device=self.device)
23
  self.tokenizer = get_tokenizer(self._clip_model)
24
 
@@ -104,7 +105,7 @@ class CLIPTransform:
104
  else:
105
  print ("Invalid request")
106
  raise Exception("Invalid request")
107
- return embeddings.cpu().numpy().tolist()
108
 
109
  request = await http_request.json()
110
  # print(type(request))
 
11
  # from clip_retrieval.clip_client import ClipClient, Modality
12
 
13
  @serve.deployment(num_replicas=6, ray_actor_options={"num_cpus": .2, "num_gpus": 0.1})
14
+ # @serve.deployment(num_replicas=3, ray_actor_options={"num_cpus": .2, "num_gpus": 0.2})
15
  class CLIPTransform:
16
  def __init__(self):
17
  # os.environ["OMP_NUM_THREADS"] = "20"
 
19
  # Load model
20
  self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
21
  self._clip_model="ViT-L/14"
22
+ # self._clip_model="open_clip:ViT-H-14"
23
  self.model, self.preprocess = load_clip(self._clip_model, use_jit=True, device=self.device)
24
  self.tokenizer = get_tokenizer(self._clip_model)
25
 
 
105
  else:
106
  print ("Invalid request")
107
  raise Exception("Invalid request")
108
+ return embeddings.float().cpu().numpy().tolist()
109
 
110
  request = await http_request.json()
111
  # print(type(request))
experimental/clip_app_client.py CHANGED
@@ -28,10 +28,11 @@ class ClipAppClient:
28
  """
29
 
30
  def __init__(self, clip_model="ViT-L/14", device=None):
 
31
  self.clip_model = clip_model
32
  self.device = device or ("cuda:0" if torch.cuda.is_available() else "cpu")
33
  print("using device", self.device)
34
- self.model, self.preprocess = load_clip(clip_model, use_jit=True, device=self.device)
35
  self.tokenizer = get_tokenizer(clip_model)
36
 
37
  def preprocess_image(self, image_url):
 
28
  """
29
 
30
  def __init__(self, clip_model="ViT-L/14", device=None):
31
+ # def __init__(self, clip_model="open_clip:ViT-H-14", device=None):
32
  self.clip_model = clip_model
33
  self.device = device or ("cuda:0" if torch.cuda.is_available() else "cpu")
34
  print("using device", self.device)
35
+ _, self.preprocess = load_clip(clip_model, use_jit=True, device=self.device)
36
  self.tokenizer = get_tokenizer(clip_model)
37
 
38
  def preprocess_image(self, image_url):
experimental/vision001.py CHANGED
@@ -12,6 +12,8 @@ from clip_retrieval.clip_client import ClipClient, Modality
12
  clip_retrieval_service_url = "https://knn.laion.ai/knn-service"
13
  map_clip_to_clip_retreval = {
14
  "ViT-L/14": "laion5B-L-14",
 
 
15
  }
16
 
17
 
 
12
  clip_retrieval_service_url = "https://knn.laion.ai/knn-service"
13
  map_clip_to_clip_retreval = {
14
  "ViT-L/14": "laion5B-L-14",
15
+ "open_clip:ViT-H-14": "laion5B-H-14",
16
+ "open_clip:ViT-L-14": "laion5B-L-14",
17
  }
18
 
19
 
experimental/vision002.py CHANGED
@@ -12,6 +12,8 @@ from clip_retrieval.clip_client import ClipClient, Modality
12
  clip_retrieval_service_url = "https://knn.laion.ai/knn-service"
13
  map_clip_to_clip_retreval = {
14
  "ViT-L/14": "laion5B-L-14",
 
 
15
  }
16
 
17
 
@@ -55,8 +57,8 @@ def clustering_templates(embeddings, n_clusters=5):
55
  return templates
56
 
57
  # test_image_path = os.path.join(os.getcwd(), "images", "plant-001.png")
58
- test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-001.jpeg")
59
- # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
60
  # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
61
  # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "car-002.jpeg")
62
 
@@ -78,6 +80,7 @@ print (f"embeddings: {preprocessed_image_embeddings.shape}")
78
 
79
 
80
  template = preprocessed_image_embeddings
 
81
  for step_num in range(3):
82
  print (f"\n\n---- Step {step_num} ----")
83
 
@@ -123,7 +126,10 @@ for step_num in range(3):
123
  # template = clusters[cluster_similarity[0][1]]
124
  template = preprocessed_image_embeddings * (len(clusters)-1)
125
  for i in range(1, len(clusters)):
126
- template -= clusters[cluster_similarity[i][1]]
 
 
 
127
  print("---")
128
  print(f"seaching based on template")
129
  results = clip_retrieval_client.query(embedding_input=template[0].tolist())
 
12
  clip_retrieval_service_url = "https://knn.laion.ai/knn-service"
13
  map_clip_to_clip_retreval = {
14
  "ViT-L/14": "laion5B-L-14",
15
+ "open_clip:ViT-H-14": "laion5B-H-14",
16
+ "open_clip:ViT-L-14": "laion5B-L-14",
17
  }
18
 
19
 
 
57
  return templates
58
 
59
  # test_image_path = os.path.join(os.getcwd(), "images", "plant-001.png")
60
+ # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-001.jpeg")
61
+ test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
62
  # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
63
  # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "car-002.jpeg")
64
 
 
80
 
81
 
82
  template = preprocessed_image_embeddings
83
+ template = template / template.norm()
84
  for step_num in range(3):
85
  print (f"\n\n---- Step {step_num} ----")
86
 
 
126
  # template = clusters[cluster_similarity[0][1]]
127
  template = preprocessed_image_embeddings * (len(clusters)-1)
128
  for i in range(1, len(clusters)):
129
+ cluster = clusters[cluster_similarity[i][1]]
130
+ normalized_cluster = cluster / cluster.norm()
131
+ template -= normalized_cluster
132
+ template = template / template.norm()
133
  print("---")
134
  print(f"seaching based on template")
135
  results = clip_retrieval_client.query(embedding_input=template[0].tolist())