first commit

.idea/FETA_IKEA.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/deployment.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" remoteFilesAllowedToDisappearOnAutoupload="false">
+    <serverData>
+      <paths name="amitalfa@cccxl014.pok.ibm.com:22 key">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+      <paths name="amitalfa@cccxl015.pok.ibm.com:22 key">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+      <paths name="dino_fork">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+      <paths name="doc-seatch-demo">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+      <paths name="nextup-common-models">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+      <paths name="open_clip_honda">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+    </serverData>
+  </component>
+</project>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/FETA_IKEA.iml" filepath="$PROJECT_DIR$/.idea/FETA_IKEA.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="6fa0fe35-6eb8-444c-86ae-e95ad54ebce6" name="Changes" comment="" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectId" id="2CR1g7GMI5SN8HD4YQlhrMqxJeT" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "RunOnceActivity.OpenProjectViewOnStart": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "WebServerToolWindowFactoryState": "true",
+    "last_opened_file_path": "C:/Users/001520756/PycharmProjects/FETA_IKEA"
+  }
+}]]></component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="C:\Users\001520756\PycharmProjects\FETA_IKEA" />
+    </key>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="6fa0fe35-6eb8-444c-86ae-e95ad54ebce6" name="Changes" comment="" />
+      <created>1658750692140</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1658750692140</updated>
+      <workItem from="1658750693739" duration="753000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+</project>

README.md

@@ -0,0 +1,17 @@
+---
+title: FETA  
+emoji: 🏢  
+colorFrom: blue  
+colorTo: purple  
+sdk: gradio  
+sdk_version: 3.0.20  
+app_file: app.py  
+pinned: false  
+license: apache-2.0  
+---  
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference  
+
+
+This is a demo for the paper: "FETA: Towards Specializing Foundational Models for Expert Task Applications"
+

app.py

@@ -0,0 +1,117 @@
+import pickle
+import os
+from sklearn.neighbors import NearestNeighbors
+import numpy as np
+import gradio as gr
+from PIL import Image
+
+data_root = '.'
+img_data_root = 'https://nextup-public-media.s3.ap.cloud-object-storage.appdomain.cloud/documents/search/ikea/'
+feat_dir = os.path.join(data_root, 'feats')
+doc_names = os.listdir(feat_dir)
+num_nn = 5
+
+# search_domain = 'all'
+# num_results_per_domain = 5
+src_data_dict = {}
+for doc_name in doc_names:
+    with open(os.path.join(feat_dir, doc_name), 'rb') as fp:
+        doc_data = pickle.load(fp)
+        t2i_space = NearestNeighbors(n_neighbors=num_nn, algorithm='auto', n_jobs=-1, metric='correlation').fit(doc_data['image_feat'])
+        i2t_space = NearestNeighbors(n_neighbors=num_nn, algorithm='auto', n_jobs=-1, metric='correlation').fit(doc_data['text_feat'])
+        src_data_dict[doc_name] = (doc_data, t2i_space, i2t_space)
+
+def query_i2t(query_index, query_doc):
+    doc_data = src_data_dict[query_doc][0]
+#     src_img_path = os.path.join(data_root, doc_data['img_paths'][query_index])
+    src_img_path = os.path.join(img_data_root,
+                                      '/'.join(doc_data['img_paths'][query_index].split('/')[-2:]))
+#     print(src_img_path)
+#     src_page_path = 'pages'.join(src_img_path.split('images'))
+#     src_page_path = '_'.join(src_page_path.split('_')[:-1])+'.png'
+    page_file_name = os.path.basename(src_img_path).split('_')[0] + f".{os.path.basename(src_img_path).split('.')[-1]}"
+    src_page_path = os.path.join(os.path.dirname(src_img_path), page_file_name)
+    _, top_n_matches_ids = src_data_dict[query_doc][2].kneighbors(doc_data['image_feat'][query_index].unsqueeze(0))
+    captions = [doc_data['texts'][i] for i in top_n_matches_ids[0]]
+    return [src_page_path] + captions
+
+def query_t2i(query_index, query_doc):
+    doc_data = src_data_dict[query_doc][0]
+    src_txt = doc_data['texts'][query_index]
+    _, top_n_matches_ids = src_data_dict[query_doc][1].kneighbors(doc_data['text_feat'][query_index].unsqueeze(0))
+    dst_image_paths = []
+    dst_page_paths = []
+    for i in range(num_nn):
+#         dst_img_path = os.path.join(data_root, doc_data['img_paths'][top_n_matches_ids[0][i]])
+        dst_img_path = os.path.join(img_data_root,
+                                      '/'.join(doc_data['img_paths'][top_n_matches_ids[0][i]].split('/')[-2:]))
+
+        dst_image_paths.append(dst_img_path)
+#         dst_page_path = 'pages'.join(dst_img_path.split('images'))
+#         dst_page_path = '_'.join(dst_page_path.split('_')[:-1])+'.png'
+        page_file_name = os.path.basename(dst_img_path).split('_')[0] + f".{os.path.basename(dst_img_path).split('.')[-1]}"
+        dst_page_path = os.path.join(os.path.dirname(dst_img_path), page_file_name)
+        dst_page_paths.append(dst_page_path)
+    return [src_txt] + dst_page_paths
+
+demo = gr.Blocks()
+with demo:
+    gr.Markdown('# FETA towards Specializing Foundational Models for Expert Task Applications')
+    gr.Markdown('This demo showcases the txt to image and image to text retrieval capabilities of FETA.')
+    gr.Markdown('The model is trained in an self-supervised automated manner on a folder of PDF documents without any manual labels.')
+    gr.Markdown('## Instructions:')
+    gr.Markdown('Select a query domain and a class from the drop-down menus and select any random image index from the domain using the slider below, then press the "Run" button. The query image and the retrieved results from each of the four domains, along with the class label will be presented.')
+    gr.Markdown('## Select Query Domain: ')
+    gr.Markdown('# Query Image: \t\t\t\t')
+    # domain_drop = gr.Dropdown(domains)
+    # cl_drop = gr.Dropdown(class_list)
+    # domain_select_button = gr.Button("Select Domain")
+    # slider = gr.Slider(0, min_len)
+    # slider = gr.Slider(0, 10000)
+    with gr.Tabs():
+        with gr.TabItem("image to text"):
+            with gr.Row():
+                with gr.Column():
+                    doc_drop_i2t = gr.Dropdown(doc_names, label='Doc name')
+                    slider_i2t = gr.Slider(0, 100, label='Query image selector slider') # TODO: make this len(doc_drop) instead
+
+                # gr.Markdown('\t')
+                # gr.Markdown('\t')
+                # gr.Markdown('\t')
+                with gr.Column():
+#                     src_img_i2t = gr.Image()
+                    src_page_i2t = gr.Image()
+            button_i2t = gr.Button("Run")
+            out_captions_i2t = []
+            gr.Markdown(f'# Retrieved texts:')
+            with gr.Row():
+                for _ in range(num_nn):
+                    with gr.Column():
+                        out_captions_i2t.append(gr.Label())
+        with gr.TabItem("text to image"):
+            with gr.Row():
+                with gr.Column():
+                    doc_drop_t2i = gr.Dropdown(doc_names, label='Doc name')
+                    slider_t2i = gr.Slider(0, 100, label='Query text selector slider') # TODO: make this len(doc_drop) instead
+
+                # gr.Markdown('\t')
+                # gr.Markdown('\t')
+                # gr.Markdown('\t')
+                with gr.Column():
+                    src_caption_t2i = gr.Text()
+
+            button_t2i = gr.Button("Run")
+            dst_images_t2i = []
+            dst_pages_t2i = []
+            gr.Markdown(f'# Retrieved images:')
+            for _ in range(num_nn):
+                with gr.Row():
+#                     with gr.Column():
+#                         dst_images_t2i.append(gr.Image())
+                    with gr.Column():
+                        dst_pages_t2i.append(gr.Image())
+
+    button_i2t.click(query_i2t, inputs=[slider_i2t, doc_drop_i2t], outputs=[src_page_i2t] + out_captions_i2t)
+    button_t2i.click(query_t2i, inputs=[slider_t2i, doc_drop_t2i], outputs= [src_caption_t2i] + dst_pages_t2i)
+
+demo.launch(share=True)

feats/OCR_1986_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26e3d8a6e540c8066fb23c288ddcc0e16f9af2aebc29db943bbc090b58627819
+size 2217299

feats/OCR_1988_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca1cd7c789ccc1f46913f91fa0c6128b467a485855e7c31679c8bcde4ccc7da
+size 3526045

feats/OCR_1989_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4c3cbc31771ab55bf62e6357e6fa91194a15f321181fbeb079e983cc52c7163
+size 4401054

feats/OCR_1990_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0877d7d581be2d90f435efc7191dd699bb9a832c8fb69471a0c42da1ffcd7540
+size 3511136

feats/OCR_1991_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c56ca742388dc24c2e4cc9f00868ccbe4f8b6a2fd945691748036e089c79dd9
+size 3200446

feats/OCR_1992_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52f18d069a24f001cb96fb7d932fd5be760eb3c801776c19a8330df8de29b982
+size 15974193

feats/OCR_1993_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:021d11c2bcf44e7ecc9cb95372884ef471f7c409e243b9a5791099440cc5bc09
+size 9268128

feats/OCR_1994_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62070c66c1d1da9a996b08cb95939c8aca38dfacd7a43466b56103ee8db858de
+size 13055739

feats/OCR_1995_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:255c18061109c77ac9dd04b13148a2671516eae82aeab514e29acff67aa052be
+size 7627396

feats/OCR_1996_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91bad4a27fc6ec5bb6501d65e3dda11cf8500f162a155603fdc04e53ba4321df
+size 9774353

feats/OCR_1997_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83bcd1e6a0c0d562b25fc131d9ec8bb2495a7d0d8352380d759d0d70b3c53589
+size 5086507

feats/OCR_1998_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dac7d60d50be1406c5ab6f4e8edd6a21c7571d7158f3743bb9a48c78c6a222e6
+size 11039538

feats/OCR_1999_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b77648d958f6b73c4037313c4371f34dfe26863d18699f54432893ba0fd3cb62
+size 8066941

feats/OCR_2000_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b7e55d1cadac42c5f9d497c25b45caa4c8e8058d557ea6dcc04538b0f8e015c
+size 16961400

feats/OCR_2001_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a67b8cd3af737126204f265c7e87bb3a0460049699aed0cc1c44ccbc9e17f9f4
+size 7879499

feats/OCR_2002_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3615dc2dd6a64def85c919acdaa61a0ba6ca4bebf1f77bc9d82cfec84375c8
+size 4546114

feats/OCR_2003_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7358021720cd5cbebdde64dc292420a5727494d73458fa0e87ead16bd33a1cfb
+size 13802401

feats/OCR_2004_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb9b730ed806d45e2ffa97407f05ec5561e3913b1e83acee3cf1c83d4dc4a0f
+size 15180789

feats/OCR_2008_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:078b0835f1a2799c36234c29c00e49144521b24d74ca056a8d089728bbca8813
+size 13416943

feats/OCR_2009_IKEA_US.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e9da5a7091dc164388e7ff714f2a6a1f1acaf1c62e1395500fdebe24460239f
+size 7292966

feats/us2010.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb78e8d96c43cddef0a015bf1a90bf7f60ae6507f09d2f793ab7cf8f27505e98
+size 8259988

feats/us2011.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ae4a38a4e948c0f56f768aee79856b5c4e66a804c2627e5881dc0924de4c7c
+size 6565430

feats/us2012.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02149bd8935413f71892a8217d071e9f5859d7cc652172e527112987f599e2da
+size 8103386

feats/us2013.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b3377fedf82ca8371c0f41513d72ebafeaf4f820803fec0d37079229f840894
+size 5398149

feats/us2014.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4cba44be476229745597c7325e5e54c5c228fd0d0a34dba4f0b7b0bf0052bfd
+size 7082986

feats/us2015.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7beb0563014d449e85a4c2acdca0cba37c72e97fccf622432d28ab832c5c70fc
+size 10220411

requirements.txt

@@ -0,0 +1,5 @@
+numpy==1.20.1
+Pillow==8.2.0
+scikit-learn==0.24.1
+
+