Upload 115 files

cashed_pickles/image_features_XTD_1000_images_XLM_Roberta_Large_Vit_B_16Plus_ar.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3986c84721b9f63ecbe86c4d9903327185865f560de8a5105f1909a5ffd54be
+size 2560139

cashed_pickles/image_features_XTD_1000_images_arabert_siglib_best_model.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ee1693fd39a670fbefba330b2ae171bd523429fccd4f6c0b5c9427a837f66a
+size 3072139

cashed_pickles/text_features_XTD_1000_images_XLM_Roberta_Large_Vit_B_16Plus_ar.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1b0bfa2ea2c8bff9da1768012c80fda84465605d56d0dcdd7ac9f63fb23f503
+size 2560139

cashed_pickles/text_features_XTD_1000_images_arabert_siglib_best_model.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd91f8e3579360094373b2874630da96d1ad29b8f7a3f81c616e3ad1ee09b969
+size 3072139

gradio_application/app.py

@@ -0,0 +1,110 @@
+import gradio as gr
+import utils
+
+
+# Araclip demo 
+with gr.Blocks() as demo_araclip:
+
+    gr.Markdown("## Input parameters")
+
+    txt = gr.Textbox(label="Text Query (Caption)")
+    num = gr.Slider(label="Number of retrieved image", value=1, minimum=1, maximum=1000)
+    
+
+    with gr.Row():
+        btn = gr.Button("Retrieve images", scale=1)
+
+    gr.Markdown("## Retrieved Images")
+
+    gallery = gr.Gallery(
+        label="Generated images", show_label=True, elem_id="gallery"
+    , columns=[5], rows=[1], object_fit="contain", height="auto")
+
+
+    with gr.Row():
+        lables = gr.Label(label="Text image similarity") 
+
+    with gr.Row():
+        
+        with gr.Column(scale=1):
+            gr.Markdown("<div style='text-align: center; font-size: 24px; font-weight: bold;'>Data Retrieved based on Images Similarity</div>")
+
+            json_output = gr.JSON()
+
+        with gr.Column(scale=1):
+            # gr.Markdown("### Data Retrieved based on Text similarity")
+            # gr.Markdown("<div style='text-align: center;'> Data Retrieved based on Text similarity </div>")
+            gr.Markdown("<div style='text-align: center; font-size: 24px; font-weight: bold;'>Data Retrieved based on Text similarity</div>")
+            json_text = gr.JSON()
+
+
+    btn.click(utils.predict, inputs=[txt, num], outputs=[gallery,lables, json_output, json_text])
+
+
+    gr.Examples(
+        examples=[["تخطي لاعب فريق بيتسبرج بايرتس منطقة اللوحة الرئيسية في مباراة بدوري البيسبول", 5], 
+                  ["وقوف قطة بمخالبها على فأرة حاسوب على المكتب", 10],
+                  ["صحن به شوربة صينية بالخضار، وإلى جانبه بطاطس مقلية وزجاجة ماء", 7]],
+        inputs=[txt, num],
+        outputs=[gallery,lables, json_output, json_text],
+        fn=utils.predict,
+        cache_examples=False,
+    )
+
+# mclip demo 
+with gr.Blocks() as demo_mclip:
+
+    gr.Markdown("## Input parameters")
+    
+    txt = gr.Textbox(label="Text Query (Caption)")
+    num = gr.Slider(label="Number of retrieved image", value=1, minimum=1, maximum=1000)
+
+    with gr.Row():
+        btn = gr.Button("Retrieve images", scale=1)
+
+    gr.Markdown("## Retrieved Images")
+
+    gallery = gr.Gallery(
+        label="Generated images", show_label=True, elem_id="gallery_mclip"
+    , columns=[5], rows=[1], object_fit="contain", height="auto")
+
+    
+    lables = gr.Label() 
+
+    with gr.Row():
+        
+        with gr.Column(scale=1):
+            gr.Markdown("## Images Retrieved")
+            json_output = gr.JSON()
+
+        with gr.Column(scale=1):
+            gr.Markdown("## Text Retrieved")
+            json_text = gr.JSON()
+
+    btn.click(utils.predict_mclip, inputs=[txt, num], outputs=[gallery,lables, json_output, json_text])
+
+    
+
+
+    gr.Examples(
+        examples=[["تخطي لاعب فريق بيتسبرج بايرتس منطقة اللوحة الرئيسية في مباراة بدوري البيسبول", 5], 
+                  ["وقوف قطة بمخالبها على فأرة حاسوب على المكتب", 10],
+                  ["صحن به شوربة صينية بالخضار، وإلى جانبه بطاطس مقلية وزجاجة ماء", 7]],
+        inputs=[txt, num],
+        outputs=[gallery,lables, json_output, json_text],
+        fn=utils.predict_mclip,
+        cache_examples=False,
+    )
+
+
+# Group the demos in a TabbedInterface 
+with gr.Blocks() as demo:
+
+    gr.Markdown("<font color=red size=10><center>AraClip: Arabic Image Retrieval Application</center></font>")
+    gr.TabbedInterface([demo_araclip, demo_mclip], ["Our Model", "Mclip model"])
+
+
+
+if __name__ == "__main__":
+    
+    demo.launch()

gradio_application/model_loading.py

@@ -0,0 +1,51 @@
+
+import pickle
+import torch
+import transformers
+import gradio as gr
+
+
+# XLM model functions 
+import transformers
+
+
+# Our model definition 
+
+class MultilingualClipEdited(torch.nn.Module):
+    def __init__(self, model_name, tokenizer_name, head_name, weights_dir='head_weights/', cache_dir=None,in_features=None,out_features=None):
+        super().__init__()
+        self.model_name = model_name
+        self.tokenizer_name = tokenizer_name
+        self.head_path = weights_dir + head_name
+
+        self.tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer_name, cache_dir=cache_dir)
+        self.transformer = transformers.AutoModel.from_pretrained(model_name, cache_dir=cache_dir)
+        self.clip_head = torch.nn.Linear(in_features=in_features, out_features=out_features)
+        self._load_head()
+
+    def forward(self, txt):
+        txt_tok = self.tokenizer(txt, padding=True, return_tensors='pt')
+        embs = self.transformer(**txt_tok)[0]
+        att = txt_tok['attention_mask']
+        embs = (embs * att.unsqueeze(2)).sum(dim=1) / att.sum(dim=1)[:, None]
+        return self.clip_head(embs)
+
+    def _load_head(self):
+        with open(self.head_path, 'rb') as f:
+            lin_weights = pickle.loads(f.read())
+        self.clip_head.weight = torch.nn.Parameter(torch.tensor(lin_weights[0]).float().t())
+        self.clip_head.bias = torch.nn.Parameter(torch.tensor(lin_weights[1]).float())
+
+AVAILABLE_MODELS = {
+    'bert-base-arabertv2-ViT-B-16-SigLIP-512-epoch-155-trained-2M':{
+    'model_name': 'Arabic-Clip/bert-base-arabertv2-ViT-B-16-SigLIP-512-epoch-155-trained-2M',
+    'tokenizer_name': 'Arabic-Clip/bert-base-arabertv2-ViT-B-16-SigLIP-512-epoch-155-trained-2M',
+    'head_name': 'arabertv2-vit-B-16-siglibheads_of_the_model_arabertv2-ViT-B-16-SigLIP-512-155_.pickle'
+    },
+
+}
+
+
+def load_model(name, cache_dir=None,in_features=None,out_features=None):
+    config = AVAILABLE_MODELS[name]
+    return MultilingualClipEdited(**config, cache_dir=cache_dir, in_features= in_features, out_features=out_features)

gradio_application/utils.py

@@ -0,0 +1,200 @@
+
+import os 
+import numpy as np
+import pickle
+import torch
+import transformers
+from PIL import Image
+from open_clip import create_model_from_pretrained, create_model_and_transforms
+import json 
+
+# XLM model functions 
+from multilingual_clip import pt_multilingual_clip
+
+from model_loading import load_model
+
+
+
+class CustomDataSet(torch.utils.data.Dataset):
+    def __init__(self, main_dir, compose, image_name_list):
+        self.main_dir = main_dir
+        self.transform = compose
+        self.total_imgs = image_name_list
+
+    def __len__(self):
+        return len(self.total_imgs)
+
+    def get_image_name(self, idx):
+
+        return self.total_imgs[idx]
+
+    def __getitem__(self, idx):
+        img_loc = os.path.join(self.main_dir, self.total_imgs[idx])
+        image = Image.open(img_loc)
+
+        return self.transform(image)
+
+
+def features_pickle(file_path=None):
+
+    with open(file_path, 'rb') as handle:
+        features_pickle = pickle.load(handle)
+
+    return features_pickle
+
+
+def dataset_loading():
+
+    with open("photos/en_ar_XTD10_edited_v2.jsonl") as filino:
+
+
+        data = [json.loads(file_i) for file_i in filino]
+
+    sorted_data = sorted(data, key=lambda x: x['id'])
+
+    image_name_list = [lin["image_name"] for lin in sorted_data]
+
+
+    return sorted_data, image_name_list 
+
+
+def text_encoder(language_model, text):
+    """Normalize the text embeddings"""
+    embedding = language_model(text)
+    norm_embedding = embedding / np.linalg.norm(embedding)
+
+    return embedding, norm_embedding
+
+
+def compare_embeddings(logit_scale, img_embs, txt_embs):
+  
+  image_features = img_embs / img_embs.norm(dim=-1, keepdim=True)
+
+  text_features = txt_embs / txt_embs.norm(dim=-1, keepdim=True)
+
+  logits_per_text = logit_scale * text_features @ image_features.t()
+
+  return logits_per_text
+
+# Done 
+def compare_embeddings_text(full_text_embds, txt_embs):
+  
+  full_text_embds_features = full_text_embds / full_text_embds.norm(dim=-1, keepdim=True)
+
+  text_features = txt_embs / txt_embs.norm(dim=-1, keepdim=True)
+
+  logits_per_text_full = text_features @ full_text_embds_features.t()
+
+  return logits_per_text_full
+
+
+
+def find_image(language_model,clip_model, text_query, dataset, image_features, text_features_new,sorted_data, num=1):
+
+    embedding, _  = text_encoder(language_model, text_query)
+
+    logit_scale = clip_model.logit_scale.exp().float().to('cpu')
+
+    language_logits, text_logits = {}, {}
+
+    language_logits["Arabic"] = compare_embeddings(logit_scale, torch.from_numpy(image_features), torch.from_numpy(embedding))
+
+    text_logits["Arabic_text"] = compare_embeddings_text(torch.from_numpy(text_features_new), torch.from_numpy(embedding))
+
+    
+    for _, txt_logits in language_logits.items():
+
+        probs = txt_logits.softmax(dim=-1).cpu().detach().numpy().T
+
+        file_paths = []
+        labels, json_data = {}, {}
+
+        for i in range(1, num+1):
+            idx = np.argsort(probs, axis=0)[-i, 0]
+            path = 'photos/XTD10_dataset/' + dataset.get_image_name(idx)
+                    
+            path_l = (path,f"{sorted_data[idx]['caption_ar']}")
+
+            labels[f" Image # {i}"] = probs[idx]
+            json_data[f" Image # {i}"] = sorted_data[idx]
+
+            file_paths.append(path_l)
+
+
+    json_text = {} 
+
+    for _, txt_logits_full in text_logits.items():
+
+        probs_text = txt_logits_full.softmax(dim=-1).cpu().detach().numpy().T
+
+        for j in range(1, num+1):
+
+            idx = np.argsort(probs_text, axis=0)[-j, 0]
+            json_text[f" Text # {j}"] = sorted_data[idx]
+
+    return file_paths, labels, json_data, json_text
+
+
+
+class AraClip():
+    def __init__(self):
+
+        self.text_model = load_model('bert-base-arabertv2-ViT-B-16-SigLIP-512-epoch-155-trained-2M', in_features= 768, out_features=768)
+        self.language_model = lambda queries: np.asarray(self.text_model(queries).detach().to('cpu')) 
+        self.clip_model, self.compose = create_model_from_pretrained('hf-hub:timm/ViT-B-16-SigLIP-512')
+        self.sorted_data, self.image_name_list = dataset_loading()
+
+    def load_images(self):
+        # Return the features of the text and images
+        image_features_new = features_pickle('cashed_pickles/image_features_XTD_1000_images_arabert_siglib_best_model.pickle')
+        return image_features_new
+    
+    def load_text(self):
+        text_features_new = features_pickle('cashed_pickles/text_features_XTD_1000_images_arabert_siglib_best_model.pickle')
+        return text_features_new
+    
+    def load_dataset(self):
+        dataset = CustomDataSet("photos/XTD10_dataset", self.compose, self.image_name_list)
+        return dataset
+
+
+araclip = AraClip()
+
+def predict(text, num):
+
+    image_paths, labels, json_data, json_text = find_image(araclip.language_model,araclip.clip_model, text, araclip.load_dataset(), araclip.load_images() , araclip.load_text(), araclip.sorted_data, num=int(num))
+
+    return image_paths, labels, json_data, json_text
+
+
+class Mclip():
+    def __init__(self) -> None:
+
+    
+        self.tokenizer_mclip = transformers.AutoTokenizer.from_pretrained('M-CLIP/XLM-Roberta-Large-Vit-B-16Plus')
+        self.text_model_mclip = pt_multilingual_clip.MultilingualCLIP.from_pretrained('M-CLIP/XLM-Roberta-Large-Vit-B-16Plus')
+        self.language_model_mclip = lambda queries:  np.asarray(self.text_model_mclip.forward(queries, self.tokenizer_mclip).detach().to('cpu'))  
+        self.clip_model_mclip, _, self.compose_mclip = create_model_and_transforms('ViT-B-16-plus-240', pretrained="laion400m_e32")
+        self.sorted_data, self.image_name_list = dataset_loading()
+
+    def load_images(self):
+        # Return the features of the text and images
+        image_features_mclip = features_pickle('cashed_pickles/image_features_XTD_1000_images_XLM_Roberta_Large_Vit_B_16Plus_ar.pickle')
+        return image_features_mclip
+    
+    def load_text(self):
+        text_features_new_mclip = features_pickle('cashed_pickles/text_features_XTD_1000_images_XLM_Roberta_Large_Vit_B_16Plus_ar.pickle')
+        return text_features_new_mclip
+    
+    def load_dataset(self):
+        dataset_mclip = CustomDataSet("photos/XTD10_dataset", self.compose_mclip, self.image_name_list)
+        return dataset_mclip
+
+
+mclip = Mclip()
+
+def predict_mclip(text, num):
+
+    image_paths, labels, json_data, json_text = find_image(mclip.language_model_mclip,mclip.clip_model_mclip, text, mclip.load_dataset() , mclip.load_text() , mclip.load_text() , mclip.sorted_data , num=int(num))
+
+    return image_paths, labels, json_data, json_text

head_weights/arabertv2-vit-B-16-siglibheads_of_the_model_arabertv2-ViT-B-16-SigLIP-512-155_.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:360d4cf756dfc96ebbe9ccaf90f943e1d25a9ca7ca2e225cb7715893c5f62fbb
+size 2362569