Upload 6 files

Base-RCNN-FPN.yml

@@ -0,0 +1,69 @@
+MODEL:
+  MASK_ON: True
+  META_ARCHITECTURE: "GeneralizedRCNN"
+  PIXEL_MEAN: [123.675, 116.280, 103.530]
+  PIXEL_STD: [58.395, 57.120, 57.375]
+  BACKBONE:
+    NAME: "build_vit_fpn_backbone"
+  VIT:
+    OUT_FEATURES: ["layer3", "layer5", "layer7", "layer11"]
+    DROP_PATH: 0.1
+    IMG_SIZE: [224,224]
+    POS_TYPE: "abs"
+  FPN:
+    IN_FEATURES: ["layer3", "layer5", "layer7", "layer11"]
+  ANCHOR_GENERATOR:
+    SIZES: [[32], [64], [128], [256], [512]]  # One size for each in feature map
+    ASPECT_RATIOS: [[0.5, 1.0, 2.0]]  # Three aspect ratios (same for all in feature maps)
+  RPN:
+    IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"]
+    PRE_NMS_TOPK_TRAIN: 2000  # Per FPN level
+    PRE_NMS_TOPK_TEST: 1000  # Per FPN level
+    # Detectron1 uses 2000 proposals per-batch,
+    # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue)
+    # which is approximately 1000 proposals per-image since the default batch size for FPN is 2.
+    POST_NMS_TOPK_TRAIN: 1000
+    POST_NMS_TOPK_TEST: 1000
+  ROI_HEADS:
+    NAME: "StandardROIHeads"
+    IN_FEATURES: ["p2", "p3", "p4", "p5"]
+    NUM_CLASSES: 5
+  ROI_BOX_HEAD:
+    NAME: "FastRCNNConvFCHead"
+    NUM_FC: 2
+    POOLER_RESOLUTION: 7
+  ROI_MASK_HEAD:
+    NAME: "MaskRCNNConvUpsampleHead"
+    NUM_CONV: 4
+    POOLER_RESOLUTION: 14
+DATASETS:
+  TRAIN: ("publaynet_train",)
+  TEST: ("publaynet_val",)
+SOLVER:
+  LR_SCHEDULER_NAME: "WarmupCosineLR"
+  AMP:
+    ENABLED: True
+  OPTIMIZER: "ADAMW"
+  BACKBONE_MULTIPLIER: 1.0
+  CLIP_GRADIENTS:
+    ENABLED: True
+    CLIP_TYPE: "full_model"
+    CLIP_VALUE: 1.0
+    NORM_TYPE: 2.0
+  WARMUP_FACTOR: 0.01
+  BASE_LR: 0.0004
+  WEIGHT_DECAY: 0.05
+  IMS_PER_BATCH: 32
+INPUT:
+  CROP:
+    ENABLED: True
+    TYPE: "absolute_range"
+    SIZE: (384, 600)
+  MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800)
+  FORMAT: "RGB"
+DATALOADER:
+  FILTER_EMPTY_ANNOTATIONS: False
+VERSION: 2
+AUG:
+  DETR: True
+SEED: 42

bibtest.ipynb

@@ -0,0 +1,195 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pdf2bib"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_metadata(file_path):\n",
+    "    pdfextractdata = pdf2bib.pdf2bib(file_path)\n",
+    "    #st.write(pdfextractdata)\n",
+    "    pdfextractdata_metadata = {} if pdfextractdata.get('metadata', {}) is None else pdfextractdata.get('metadata', {})\n",
+    "\n",
+    "    return pdfextractdata_metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[pdf2bib]: Trying to extract data to generate the BibTeX entry for the file: boiteau-et-al-2024-relating-molecular-properties-to-the-persistence-of-marine-dissolved-organic-matter-with-liquid.pdf\n",
+      "[pdf2bib]: Calling pdf2doi...\n",
+      "[pdf2doi]: Trying to retrieve a DOI/identifier for the file: boiteau-et-al-2024-relating-molecular-properties-to-the-persistence-of-marine-dissolved-organic-matter-with-liquid.pdf\n",
+      "[pdf2doi]: Method #1: Looking for a valid identifier in the document infos...\n",
+      "[pdf2doi]: Validating the possible DOI 10.1021/acs.est.3c08245 via a query to dx.doi.org...\n",
+      "[pdf2doi]: The DOI 10.1021/acs.est.3c08245 is validated by dx.doi.org.\n",
+      "[pdf2doi]: A valid DOI was found in the document info labelled '/prism:doi'.\n",
+      "[pdf2bib]: pdf2doi found a valid identifier for this paper.\n",
+      "[pdf2bib]: Parsing the info returned by dx.doi.org...\n",
+      "[pdf2bib]: A valid BibTeX entry was generated.\n"
+     ]
+    }
+   ],
+   "source": [
+    "a = pdf2bib.pdf2bib(\"boiteau-et-al-2024-relating-molecular-properties-to-the-persistence-of-marine-dissolved-organic-matter-with-liquid.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'@article{boiteau2024relating,\\n\\ttitle = {Relating Molecular Properties to the Persistence of Marine Dissolved Organic Matter with Liquid Chromatography–Ultrahigh-Resolution Mass Spectrometry},\\n\\tpublisher = {American Chemical Society (ACS)},\\n\\turl = {http://dx.doi.org/10.1021/acs.est.3c08245},\\n\\tdoi = {10.1021/acs.est.3c08245},\\n\\tjournal = {Environmental Science & Technology},\\n\\tyear = {2024},\\n\\tmonth = {2},\\n\\tauthor = {Rene M. Boiteau and Yuri E. Corilo and William R. Kew and Christian Dewey and Maria Cristina Alvarez Rodriguez and Craig A. Carlson and Tim M. Conway}\\n}'"
+      ]
+     },
+     "execution_count": 47,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "a.get(\"bibtex\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import bibtexparser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parser = bibtex.Parser()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parser = bibtexparser.bparser.BibTexParser(common_strings=True)\n",
+    "bib_database = bibtexparser.loads(a.get(\"bibtex\"), parser=parser)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def format_author_names(authors_str):\n",
+    "    authors = authors_str.split(' and ')\n",
+    "    formatted_authors = []\n",
+    "    for author in authors:\n",
+    "        parts = author.split()\n",
+    "        if len(parts) == 2:  # Simple case: First Last\n",
+    "            last, first = parts[1], parts[0]\n",
+    "            formatted_authors.append(f\"{last}, {first[0]}.\")\n",
+    "        elif len(parts) > 2:  # Handling middle names or initials\n",
+    "            last = parts[-1]\n",
+    "            initials = ''.join(f\"{part[0]}.\" for part in parts[:-1])\n",
+    "            formatted_authors.append(f\"{last}, {initials}\")\n",
+    "    if len(formatted_authors) > 1:\n",
+    "        formatted_authors_str = ', '.join(formatted_authors[:-1]) + ', & ' + formatted_authors[-1]\n",
+    "    else:\n",
+    "        formatted_authors_str = formatted_authors[0]\n",
+    "    return formatted_authors_str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def format_apa(entry):\n",
+    "    author = format_author_names(entry.get('author', ''))\n",
+    "    year = entry.get('year', '')\n",
+    "    title = entry.get('title', '')\n",
+    "    journal = entry.get('journal', '')\n",
+    "    volume = entry.get('volume', '')\n",
+    "    issue = entry.get('issue', '')\n",
+    "    pages = entry.get('page', '').replace('-', '–')  # En dash for page range\n",
+    "    doi = entry.get('doi', '')\n",
+    "    \n",
+    "    # Constructing the citation\n",
+    "    apa_citation = f\"{author} {title}. {journal} {volume}, {pages} ({year}). https://doi.org/{doi}\"\n",
+    "    return apa_citation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Boiteau, R.M., Corilo, Y.E., Kew, W.R., Dewey, C., Rodriguez, M.C.A., Carlson, C.A., & Conway, T.M. Relating Molecular Properties to the Persistence of Marine Dissolved Organic Matter with Liquid Chromatography–Ultrahigh-Resolution Mass Spectrometry. Environmental Science & Technology ,  (2024). https://doi.org/10.1021/acs.est.3c08245'"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "format_apa(bib_database.entries[0])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

cascade_dit_base.yml

@@ -0,0 +1,20 @@
+_BASE_: "Base-RCNN-FPN.yml"
+MODEL:
+  PIXEL_MEAN: [ 127.5, 127.5, 127.5 ]
+  PIXEL_STD: [ 127.5, 127.5, 127.5 ]
+  WEIGHTS: "https://layoutlm.blob.core.windows.net/dit/dit-pts/dit-base-224-p16-500k-62d53a.pth"
+  VIT:
+    NAME: "dit_base_patch16"
+  ROI_HEADS:
+    NAME: CascadeROIHeads
+  ROI_BOX_HEAD:
+    CLS_AGNOSTIC_BBOX_REG: True
+  RPN:
+    POST_NMS_TOPK_TRAIN: 2000
+SOLVER:
+  WARMUP_ITERS: 1000
+  IMS_PER_BATCH: 16
+  MAX_ITER: 60000
+  CHECKPOINT_PERIOD: 2000
+TEST:
+  EVAL_PERIOD: 2000

imagesummary_fun.py

@@ -0,0 +1,56 @@
+import base64
+import requests
+import os 
+
+from dotenv import load_dotenv
+load_dotenv()  # This loads the variables from .env
+openai_api_key = os.getenv('openai_api_key')
+#openai_api_key = os.getenv('openai_api_key')
+# Mock function to simulate image encoding and API call
+def encode_image_to_base64(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+    
+
+
+# Function to get summary from OpenAI GPT-4 Vision API
+def get_image_summary(image_path):
+    # Encode the selected image
+    base64_image = encode_image_to_base64(image_path)
+
+    # OpenAI API URL and Key
+    api_url = "https://api.openai.com/v1/chat/completions"
+    #openai_api_key = "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"
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {openai_api_key}"
+    }
+
+    payload = {
+        "model": "gpt-4-vision-preview",  # Update this if the model name changes
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{base64_image}"
+                        }
+                    },
+                    {
+                        "type": "text",
+                        "text": "You have provide an explanation for this figure or table. Consider elements like panels, axis, data and labels and etc."
+                    }
+                ]
+            }
+        ],
+        "max_tokens": 1000
+    }
+
+    response = requests.post(api_url, headers=headers, json=payload)
+    if response.status_code == 200:
+        return response.json()["choices"][0]["message"]["content"]
+    else:
+        return "Failed to get summary. Please try again."

pdfextract_fun.py

@@ -0,0 +1,192 @@
+import warnings
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
+# Filter warnings about inputs not requiring gradients
+warnings.filterwarnings("ignore", message="None of the inputs have requires_grad=True. Gradients will be None")
+warnings.filterwarnings("ignore", message="torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument.")
+
+import cv2
+import os
+import fitz  # PyMuPDF
+import numpy as np
+import re
+import pytesseract
+from PIL import Image
+from tqdm import tqdm 
+
+from unilm.dit.object_detection.ditod import add_vit_config
+
+from detectron2.config import CfgNode as CN
+from detectron2.config import get_cfg
+from detectron2.utils.visualizer import ColorMode, Visualizer
+from detectron2.data import MetadataCatalog
+from detectron2.engine import DefaultPredictor
+
+
+# Step 1: instantiate config
+cfg = get_cfg()
+add_vit_config(cfg)
+cfg.merge_from_file("cascade_dit_base.yml")
+
+# Step 2: add model weights URL to config
+cfg.MODEL.WEIGHTS = "publaynet_dit-b_cascade.pth"
+
+# Step 3: set device
+#cfg.MODEL.DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+cfg.MODEL.DEVICE =  "cuda"
+
+# Step 4: define model
+predictor = DefaultPredictor(cfg)
+
+def analyze_image(img):
+    
+    md = MetadataCatalog.get(cfg.DATASETS.TEST[0])
+    if cfg.DATASETS.TEST[0]=='icdar2019_test':
+        md.set(thing_classes=["table"])
+    else:
+        md.set(thing_classes=["text","title","list","table","figure"])
+    
+    output = predictor(img)["instances"]
+    v = Visualizer(img[:, :, ::-1],
+                    md,
+                    scale=1.0,
+                    instance_mode=ColorMode.SEGMENTATION)
+    result = v.draw_instance_predictions(output.to("cpu"))
+    result_image = result.get_image()[:, :, ::-1]
+    
+    return result_image, output, v
+
+
+
+def convert_pdf_to_jpg(pdf_path, output_folder, zoom_factor=2):
+    doc = fitz.open(pdf_path)
+    for page_num in range(len(doc)):
+        page = doc.load_page(page_num)
+
+        # Adjust zoom factor for higher resolution
+        mat = fitz.Matrix(zoom_factor, zoom_factor)  # Create a Matrix with the zoom factor
+        pix = page.get_pixmap(matrix=mat)  # Render the page using the matrix
+
+        output_file = f"{output_folder}/page_{page_num}.jpg"
+        pix.save(output_file)
+
+
+
+def process_jpeg_images(output_folder):
+     for page_num in tqdm(range(len(os.listdir(output_folder))), desc="Processing the pdf"):
+         file_path = f"{output_folder}/page_{page_num}.jpg"
+         img = cv2.imread(file_path)
+         if img is None:
+             print(f"Failed to read {file_path}. Skipping.")
+             continue
+         result_image, output, v = analyze_image(img)
+
+         # Saving logic
+         save_extracted_instances(img, output, page_num,output_folder)
+
+
+
+def save_extracted_instances(img, output, page_num, dest_folder, confidence_threshold=0.8):
+    class_names = {
+        0: "text",
+        1: "title",
+        2: "list",
+        3: "table",
+        4: "figure"
+    }
+
+    threshold_value = 0  # Standard deviation threshold
+    min_height = 0  # Minimum height threshold
+
+    instances = output.to("cpu")
+    boxes = instances.pred_boxes.tensor.numpy()
+    class_ids = instances.pred_classes.tolist()
+    scores = instances.scores.tolist()  # Get prediction scores
+
+    image_counter = 1
+    for box, class_id, score in zip(boxes, class_ids, scores):
+        # Check if the prediction score meets the confidence threshold
+        if score >= confidence_threshold:
+            class_name = class_names.get(class_id, "unknown")
+
+            # Save only if class is 'figure' or 'table'
+            if class_name in ["figure", "table","text"]:
+                x1, y1, x2, y2 = map(int, box)
+                cropped_image = img[y1:y2, x1:x2]
+
+                if np.std(cropped_image) > threshold_value and (y2 - y1) > min_height:
+                    save_path = os.path.join(dest_folder, f"page_{page_num}_{class_name}_{image_counter}.jpg")
+                    cv2.imwrite(save_path, cropped_image)
+                    image_counter += 1
+
+
+def delete_files_in_folder(folder_path):
+    for filename in os.listdir(folder_path):
+        file_path = os.path.join(folder_path, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+
+
+def rename_files_sequentially(folder_path):
+    # Regex pattern to match 'page_{page_num}_{class_name}_{image_counter}.jpg'
+    pattern = re.compile(r'page_(\d+)_(\w+)_(\d+).jpg', re.IGNORECASE)
+
+    # List files in the folder
+    files = os.listdir(folder_path)
+
+    # Filter and sort files based on the regex pattern
+    sorted_files = sorted(
+        [f for f in files if pattern.match(f)],
+        key=lambda x: (int(pattern.match(x).group(1)), pattern.match(x).group(2).lower(), int(pattern.match(x).group(3)))
+    )
+
+    # Initialize an empty dictionary for counters
+    counters = {}
+
+    for filename in sorted_files:
+        match = pattern.match(filename)
+        if match:
+            page_num, class_name, _ = match.groups()
+            class_name = class_name.lower()  # Convert class name to lowercase
+
+            # Initialize counter for this class if it doesn't exist
+            if class_name not in counters:
+                counters[class_name] = 1
+
+            # New filename format: '{class_name}_{sequential_number}.jpg'
+            new_filename = f"{class_name}_{counters[class_name]}.jpg"
+            counters[class_name] += 1
+
+            # Rename the file
+            os.rename(os.path.join(folder_path, filename), os.path.join(folder_path, new_filename))
+
+            #print(f"Renamed '{filename}' to '{new_filename}'")
+
+
+def ocr_folder(folder_path):
+    # Regex pattern to match 'text_{number}.jpg'
+    pattern = re.compile(r'text_\d+\.jpg', re.IGNORECASE)
+
+    # Create a subfolder for the OCR text files
+    ocr_text_folder = os.path.join(folder_path, "ocr_results")
+    if not os.path.exists(ocr_text_folder):
+        os.makedirs(ocr_text_folder)
+
+    for filename in os.listdir(folder_path):
+        if pattern.match(filename):
+            image_path = os.path.join(folder_path, filename)
+            text = ocr_image(image_path)
+            
+            # Save the OCR result to a text file in the subfolder
+            text_file_name = filename.replace('.jpg', '.txt')
+            text_file_path = os.path.join(ocr_text_folder, text_file_name)
+            with open(text_file_path, 'w') as file:
+                file.write(text)
+            
+            #print(f"OCR result for {filename} saved to {text_file_path}\n")
+
+def ocr_image(image_path):
+    image = Image.open(image_path)
+    text = pytesseract.image_to_string(image)
+    return text

pdfsummary_fun.py

@@ -0,0 +1,61 @@
+from langchain_community.document_loaders import DirectoryLoader, TextLoader
+from langchain_openai import OpenAIEmbeddings
+from sklearn.cluster import KMeans
+import numpy as np
+from sklearn.decomposition import PCA
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI
+import os 
+
+
+from dotenv import load_dotenv
+load_dotenv()  # This loads the variables from .env
+openai_api_key = os.getenv('openai_api_key')
+
+def pdf_summary(ocr_results_folder):
+
+
+    #openai_api_key = "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"
+    loader = DirectoryLoader(ocr_results_folder, glob="**/*.txt", loader_cls=TextLoader)
+
+    docs = loader.load()
+    page_contents = [doc.page_content for doc in docs]
+
+    embeddings_model = OpenAIEmbeddings(model="text-embedding-3-small",openai_api_key=openai_api_key)
+    embeddings = embeddings_model.embed_documents(page_contents)
+
+    X = np.array(embeddings)
+    num_clusters = 20
+    kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(X)
+    pca = PCA(n_components=2)
+    X_pca = pca.fit_transform(X)
+    centroids = kmeans.cluster_centers_
+    centroids_pca = pca.transform(centroids)
+
+    closest_point_indices = find_closest_point_indices(X, centroids, 1)
+    extracted_contents = [page_contents[index[0]] for index in closest_point_indices[:num_clusters]]
+
+    prompt = ChatPromptTemplate.from_template("Summarize the article based on the texts provided from four aspects: Goal, Method, Results, and Conclusion: {topic}")
+    model = ChatOpenAI(model="gpt-3.5-turbo", openai_api_key=openai_api_key)
+    output_parser = StrOutputParser()
+
+    chain = prompt | model | output_parser
+
+    results = chain.invoke({"topic": ' '.join(extracted_contents)})
+
+    return results
+
+def find_closest_point_indices(X, centroids, num_points=1):
+    closest_indices = []
+    for center in centroids:
+        # Calculating Euclidean distances from each point in X to the centroid
+        distances = np.linalg.norm(X - center, axis=1)
+
+        # Getting the indices of the closest 'num_points' points
+        closest_idx = np.argsort(distances)[:num_points]
+
+        # Adding the indices of the closest points for this centroid
+        closest_indices.append(closest_idx)
+
+    return closest_indices