Add application file

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+cache.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,165 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+.python-version
+
+# cache.json
+data/

Dockerfile

@@ -0,0 +1,34 @@
+FROM python:3.10-slim
+
+RUN apt-get update && \
+    apt-get install git gsutil -y && \
+    apt clean && \
+    rm -rf /var/cache/apt/*
+
+WORKDIR /code
+
+COPY requirements.txt /code/requirements.txt
+
+# PYTHONDONTWRITEBYTECODE=1: Disables the creation of .pyc files (compiled bytecode)
+# PYTHONUNBUFFERED=1: Disables buffering of the standard output stream
+# PYTHONIOENCODING: specifies the encoding to be used for the standard input, output, and error streams
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=utf-8
+
+RUN pip install -U pip && \
+    pip install --no-cache-dir -r /code/requirements.txt && \
+    python -m nltk.downloader -d /usr/share/nltk_data wordnet
+
+RUN useradd -m -u 1000 user
+
+USER user
+
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["python", "main.py"]

cache.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db9e7ad9f9620753805e23353fcd3f9a972ae098d7678510949b861e598e7b28
+size 75224556

candidate.py

@@ -0,0 +1,13 @@
+from frame import Frame 
+
+class Candidate:
+    def __init__(self, frame: Frame, score: float):
+        self.frame = frame
+        self.score = score
+
+    def serialize(self) -> dict:
+        return {
+            'video': self.frame.video,
+            'frame_name': self.frame.frame_name + '.jpg',
+            'score': self.score,
+        }

classes.json

@@ -0,0 +1,83 @@
+{
+    "0": "_background_",
+    "1": "person",
+    "2": "bicycle",
+    "3": "car",
+    "4": "motorcycle",
+    "5": "airplane",
+    "6": "bus",
+    "7": "train",
+    "8": "truck",
+    "9": "boat",
+    "10": "traffic light",
+    "11": "fire hydrant",
+    "12": "stop sign",
+    "13": "parking meter",
+    "14": "bench",
+    "15": "bird",
+    "16": "cat",
+    "17": "dog",
+    "18": "horse",
+    "19": "sheep",
+    "20": "cow",
+    "21": "elephant",
+    "22": "bear",
+    "23": "zebra",
+    "24": "giraffe",
+    "25": "backpack",
+    "26": "umbrella",
+    "27": "handbag",
+    "28": "tie",
+    "29": "suitcase",
+    "30": "frisbee",
+    "31": "skis",
+    "32": "snowboard",
+    "33": "sports ball",
+    "34": "kite",
+    "35": "baseball bat",
+    "36": "baseball glove",
+    "37": "skateboard",
+    "38": "surfboard",
+    "39": "tennis racket",
+    "40": "bottle",
+    "41": "wine glass",
+    "42": "cup",
+    "43": "fork",
+    "44": "knife",
+    "45": "spoon",
+    "46": "bowl",
+    "47": "banana",
+    "48": "apple",
+    "49": "sandwich",
+    "50": "orange",
+    "51": "broccoli",
+    "52": "carrot",
+    "53": "hot dog",
+    "54": "pizza",
+    "55": "donut",
+    "56": "cake",
+    "57": "chair",
+    "58": "couch",
+    "59": "potted plant",
+    "60": "bed",
+    "61": "dining table",
+    "62": "toilet",
+    "63": "tv",
+    "64": "laptop",
+    "65": "mouse",
+    "66": "remote",
+    "67": "keyboard",
+    "68": "cell phone",
+    "69": "microwave",
+    "70": "oven",
+    "71": "toaster",
+    "72": "sink",
+    "73": "refrigerator",
+    "74": "book",
+    "75": "clock",
+    "76": "vase",
+    "77": "scissors",
+    "78": "teddy bear",
+    "79": "hair drier",
+    "80": "toothbrush"
+}

frame.py

@@ -0,0 +1,17 @@
+class Frame:
+    def __init__(self, video: str|None = None, frame_name: str|None = None, id: str|None = None) -> None:
+        if id == None:
+            self.video = video
+            self.frame_name = frame_name
+            self.id = video + '/' + frame_name
+        else:
+            self.id = id
+            self.video, self.frame_name = self.id.split('/')
+    
+    def serialize(self) -> dict:
+        return {
+            'video': self.video,
+            'frame_name': self.frame_name,
+            'id': self.id,
+        }
+    

helper.py

@@ -0,0 +1,49 @@
+from nltk.corpus import wordnet as wn
+from fastapi.responses import JSONResponse
+from google.cloud import storage
+
+OBJECT_MAP = {
+    "stop_sign": "traffic_control",
+    "sports_ball": "ball",
+    "wine_glass": "glass",
+    "potted_plant": "plant",   
+}
+
+MULTI_OBJECT_BONUS = 1.0
+
+def get_hypernym_path(object):
+    object = object.lower().replace(' ', '_')
+    if object in OBJECT_MAP:
+        object = OBJECT_MAP[object]
+    synset = wn.synsets(object, 'n')
+    if len(synset) == 0:
+        hypernym_path = [object]
+    else:
+        hypernym_path = [s.name()[:-5] for s in synset[0].hypernym_paths()[0]]
+    return hypernym_path
+
+def parse_query(query: str) -> list[dict]:
+    result = []
+    query = query.split(',')
+    for q in query:
+        q = q.split(':')
+        if len(q) == 2:
+            result.append({'object': q[0].strip(), 'amount': int(q[1].strip())})
+        else:
+            result.append({'object': q[0].strip(), 'amount': 'any'})
+    return result
+
+def make_response(status: int, message: str, data: dict|None = None) -> dict:
+    return JSONResponse(content={
+        'status': status,
+        'message': message,
+        'data': data,    
+    })
+    
+def download_from_bucket(dir: str) -> None:
+    client = storage.Client()
+    bucket = client.get_bucket('thangtd1')
+    blobs = bucket.list_blobs(prefix='object-detection')
+    for blob in blobs:
+        if blob.name.endswith('.json'):
+            blob.download_to_filename(dir + '/' + blob.name[16:])

main.py

@@ -0,0 +1,85 @@
+from searcher import Searcher
+from trie import Trie
+from helper import parse_query, make_response, download_from_bucket
+
+from fastapi import FastAPI, Request, status
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import RedirectResponse
+from pydantic import BaseModel
+
+import settings
+import os
+import time
+
+trie = Trie()
+searcher = Searcher(trie)
+app = FastAPI(title="Object Search")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_headers=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    details = exc.errors()
+    error_details = []
+
+    for error in details:
+        error_details.append({"error": f"{error['msg']} {str(error['loc'])}"})
+
+    return make_response(status=200, message="Bad Request", data=error_details)
+
+
+@app.get("/", include_in_schema=False)
+async def root() -> None:
+    return RedirectResponse("/docs")
+
+
+@app.get("/health", status_code=status.HTTP_200_OK, tags=["health"])
+async def perform_healthcheck() -> None:
+    return make_response(status=200, message="OK")
+
+
+class Query(BaseModel):
+    query_text: str
+    topk: int
+
+
+@app.post("/search", status_code=status.HTTP_200_OK, tags=["search"])
+async def search(query: Query) -> None:
+    topk = query.topk
+    query = parse_query(query.query_text)
+
+    candidates = searcher.search(query, topk)
+    data = [candidate.serialize() for candidate in candidates]
+    return make_response(status=200, message="OK", data=data)
+
+
+@app.on_event("startup")
+async def startup_event():
+    if os.path.exists("cache.json"):
+        start_time = time.time()
+        trie.load_from_cache("cache.json")
+        print("Load from cache took %.2f seconds" % (time.time() - start_time))
+    else:
+        if not os.path.exists("data"):
+            os.mkdir("data")
+            start_time = time.time()
+            download_from_bucket("data")
+            print("Download from bucket took %.2f seconds" % (time.time() - start_time))
+        start_time = time.time()
+        trie.load_from_dir("data")
+        trie.save_to_cache("cache.json")
+        print("Load from directory took %.2f seconds" % (time.time() - start_time))
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("main:app", host=settings.HOST, port=settings.PORT, reload=True)

requirements.txt

@@ -0,0 +1,35 @@
+annotated-types==0.6.0
+anyio==3.7.1
+beautifulsoup4==4.12.2
+cachetools==5.3.1
+certifi==2023.7.22
+charset-normalizer==3.3.0
+click==8.1.7
+fastapi==0.104.0
+google-api-core==2.12.0
+google-auth==2.23.3
+google-cloud-core==2.3.3
+google-cloud-storage==2.12.0
+google-crc32c==1.5.0
+google-resumable-media==2.6.0
+googleapis-common-protos==1.61.0
+h11==0.14.0
+idna==3.4
+joblib==1.3.2
+nltk==3.8.1
+numpy==1.26.1
+protobuf==4.24.4
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==2.4.2
+pydantic_core==2.10.1
+regex==2023.10.3
+requests==2.31.0
+rsa==4.9
+sniffio==1.3.0
+soupsieve==2.5
+starlette==0.27.0
+tqdm==4.66.1
+typing_extensions==4.8.0
+urllib3==2.0.7
+uvicorn==0.23.2

searcher.py

@@ -0,0 +1,45 @@
+from candidate import Candidate
+from frame import Frame
+from helper import MULTI_OBJECT_BONUS, get_hypernym_path
+
+from nltk.corpus import wordnet as wn
+
+
+class Searcher:
+    def __init__(self, trie):
+        self.trie = trie
+        
+    def search(self, query: list[dict[str, str]], topk: int) -> list[Candidate]:
+        candidates: dict[str, float] = {}
+        for q in query:
+            this_candidates: list[Candidate] = []
+            object, amount = q['object'], q['amount']
+            hypernym_path = get_hypernym_path(object)
+            node_frames = self.trie.search(hypernym_path)
+            if amount == 'any':
+                this_candidates.extend([Candidate(node_frame.frame, node_frame.p_total) for node_frame in node_frames])
+            elif amount == int(amount):
+                this_candidates.extend([Candidate(node_frame.frame, node_frame.p_of(amount) * amount) for node_frame in node_frames])
+            else:
+                raise ValueError('Amount must be an integer or "any"')
+            for candidate in this_candidates:
+                if candidate.frame.id not in candidates:
+                    candidates[candidate.frame.id] = candidate.score
+                else: 
+                    candidates[candidate.frame.id] += candidate.score + MULTI_OBJECT_BONUS
+                
+        candidates = [Candidate(Frame(id=id), score) for id, score in candidates.items()]
+        candidates = sorted(candidates, key=lambda candidate: candidate.score, reverse=True)
+        if len(candidates) > topk:
+            candidates = candidates[:topk]
+        return candidates
+        
+
+            
+        
+                
+            
+        
+        
+        
+        

settings.py

@@ -0,0 +1,2 @@
+HOST = "0.0.0.0"
+PORT = 7860

trie.py

@@ -0,0 +1,109 @@
+from frame import Frame 
+from helper import OBJECT_MAP, get_hypernym_path
+
+from numpy.polynomial import polynomial
+from nltk.corpus import wordnet as wn
+import json
+import os
+
+class NodeFrame:
+    def __init__(self, frame: Frame, p_list: list[float]) -> None:
+        self.frame = frame
+        self.p_list = p_list
+        self.p_total = self.calculate_p_total(p_list)
+        self.p_exactly = self.calculate_p_exactly(p_list)
+        
+    def calculate_p_total(self, p_list: list[float]) -> float:
+        return sum(p_list)
+
+    def calculate_p_exactly(self, p_list: list[float]) -> list[float]:
+        result = [1]
+        p_list = [[1 - p, p] for p in p_list]
+        for p in p_list:
+            result = polynomial.polymul(result, p)
+        return list(result) 
+    
+    def p_of(self, amount: int) -> float:
+        if amount < len(self.p_exactly):
+            return self.p_exactly[amount]
+        else:
+            return self.p_exactly[-1] * (0.1 ** (amount - len(self.p_exactly) + 1))
+        
+    def serialize(self) -> dict:
+        return {
+            'frame': self.frame.serialize(),
+            'p_list': self.p_list,
+        }
+
+class Node:
+    def __init__(self, node_frames: list[NodeFrame]) -> None:
+        self.node_frames = node_frames
+        self.children = {}
+
+class Trie:
+    def __init__(self) -> None:
+        self.root = Node([])
+        
+    def insert(self, node_frame: NodeFrame, path: list[str]) -> None:
+        node = self.root
+        for word in path:
+            if word not in node.children:
+                node.children[word] = Node([])
+            node = node.children[word]
+        node.node_frames.append(node_frame)
+        
+    def search(self, path: list[str]) -> list[NodeFrame]:
+        node = self.root
+        for word in path:
+            if word not in node.children:
+                return []
+            node = node.children[word]
+        return self.search_all_children(node)
+    
+    def search_all_children(self, node: Node) -> list[NodeFrame]:
+        result = []
+        if len(node.node_frames) > 0:
+            result.extend(node.node_frames)
+        for child in node.children.values():
+            result.extend(self.search_all_children(child))
+        return result
+            
+    def load_from_dir(self, dir: str) -> None:
+        for path, _, files in os.walk(dir):
+            for file in files:
+                if file.endswith('.json'):
+                    data = json.load(open(os.path.join(path, file)))
+                    video = file[:-5]
+                    for frame_name, frame_data in data.items():
+                        for object, p_list in frame_data.items():
+                            hypernym_path = get_hypernym_path(object)
+                            self.insert(NodeFrame(Frame(video=video, frame_name=frame_name), p_list), hypernym_path)
+    
+    def save_to_cache(self, cache_path: str) -> None:
+        json.dump(self.serialize(), open(cache_path, 'w'))
+        
+    def load_from_cache(self, cache_path: str) -> None:
+        self.deserialize(json.load(open(cache_path)))
+        
+    def serialize(self) -> dict:
+        output = {}
+        def dfs(node: Node, path: list[str]) -> None:
+            if len(node.node_frames) > 0:
+                output['/'.join(path)] = [node_frame.serialize() for node_frame in node.node_frames]
+            for word, child in node.children.items():
+                dfs(child, path + [word])
+        dfs(self.root, [])
+        return output
+        
+    def deserialize(self, input):
+        for path, node_frames in input.items():
+            path = path.split('/')
+            for node_frame in node_frames:
+                self.insert(NodeFrame(Frame(id=node_frame['frame']['id']), node_frame['p_list']), path)
+        
+            
+        
+    
+            
+        
+