Duplicate from Singularity666/RadiXGPT_

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: RadiXGPT
+emoji: 🏢
+colorFrom: purple
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.19.0
+app_file: app.py
+pinned: false
+license: bigscience-openrail-m
+duplicated_from: Singularity666/RadiXGPT_
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,140 @@
+import streamlit as st
+import pickle
+import pandas as pd
+import torch
+from PIL import Image
+import numpy as np
+from main import predict_caption, CLIPModel, get_text_embeddings
+import openai
+import base64
+from docx import Document
+from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
+from io import BytesIO
+import re
+
+openai.api_key = "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"
+
+st.markdown(
+    """
+<style>
+    body {
+        background-color: transparent;
+    }
+    .container {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        background-color: rgba(255, 255, 255, 0.7);
+        border-radius: 15px;
+        padding: 20px;
+    }
+    .stApp {
+        background-color: transparent;
+    }
+    .stText, .stMarkdown, .stTextInput>label, .stButton>button>span {
+        color: #1c1c1c !important; /* Set the dark text color for text elements */
+    }
+    .stButton>button>span {
+        color: initial !important; /* Reset the text color for the 'Generate Caption' button */
+    }
+    .stMarkdown h1, .stMarkdown h2 {
+        color: #ff6b81 !important; /* Set the text color of h1 and h2 elements to soft red-pink */
+        font-weight: bold; /* Set the font weight to bold */
+        border: 2px solid #ff6b81; /* Add a bold border around the headers */
+        padding: 10px; /* Add padding to the headers */
+        border-radius: 5px; /* Add border-radius to the headers */
+    }
+</style>
+""",
+    unsafe_allow_html=True,
+)
+
+device = torch.device("cpu")
+
+testing_df = pd.read_csv("testing_df.csv")
+model = CLIPModel().to(device)
+model.load_state_dict(torch.load("weights.pt", map_location=torch.device('cpu')))
+text_embeddings = torch.load('saved_text_embeddings.pt', map_location=device)
+
+def download_link(content, filename, link_text):
+    b64 = base64.b64encode(content).decode()
+    href = f'<a href="data:application/octet-stream;base64,{b64}" download="{filename}">{link_text}</a>'
+    return href
+
+def show_predicted_caption(image, top_k=8):
+    matches = predict_caption(
+        image, model, text_embeddings, testing_df["caption"]
+    )[:top_k]
+    cleaned_matches = [re.sub(r'\s\(ROCO_\d+\)', '', match) for match in matches]  # Add this line to clean the matches
+    return cleaned_matches  # Return the cleaned_matches instead of matches
+
+def generate_radiology_report(prompt):
+    response = openai.Completion.create(
+        engine="text-davinci-003",
+        prompt=prompt,
+        max_tokens=800,
+        n=1,
+        stop=None,
+        temperature=1,
+    )
+    report = response.choices[0].text.strip()
+    # Remove reference string from the report
+    report = re.sub(r'\(ROCO_\d+\)', '', report).strip()
+    return report
+
+
+def save_as_docx(text, filename):
+    document = Document()
+    document.add_paragraph(text)
+    with BytesIO() as output:
+        document.save(output)
+        output.seek(0)
+        return output.getvalue()
+
+st.title("RadiXGPT: An Evolution of machine doctors towards Radiology")
+
+# Collect user's personal information
+st.subheader("Personal Information")
+first_name = st.text_input("First Name")
+last_name = st.text_input("Last Name")
+age = st.number_input("Age", min_value=0, max_value=120, value=25, step=1)
+gender = st.selectbox("Gender", ["Male", "Female", "Other"])
+
+st.write("Upload Scan to get Radiological Report:")
+uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "png", "jpeg"])
+if uploaded_file is not None:
+    image = Image.open(uploaded_file)
+    st.image(image, caption="Uploaded Image", use_column_width=True)
+    st.write("")
+
+    if st.button("Generate Caption"):
+        with st.spinner("Generating caption..."):
+            image_np = np.array(image)
+            caption = show_predicted_caption(image_np)[0]
+
+            st.success(f"Caption: {caption}")
+
+            # Generate the radiology report
+            radiology_report = generate_radiology_report(f"Write Complete Radiology Report for this with clinical info, subjective, Assessment, Finding, Impressions, Conclusion and more in proper order : {caption}")
+
+            # Add personal information to the radiology report
+            radiology_report_with_personal_info = f"Patient Name: {first_name} {last_name}\nAge: {age}\nGender: {gender}\n\n{radiology_report}"
+
+            st.header("Radiology Report")
+            st.write(radiology_report_with_personal_info)
+            st.markdown(download_link(save_as_docx(radiology_report_with_personal_info, "radiology_report.docx"), "radiology_report.docx", "Download Report as DOCX"), unsafe_allow_html=True)
+
+        feedback_options = ["Satisfied", "Not Satisfied"]
+        selected_feedback = st.radio("Please provide feedback on the generated report:", feedback_options)
+
+        if selected_feedback == "Not Satisfied":
+            if st.button("Regenerate Report"):
+                with st.spinner("Regenerating report..."):
+                    alternative_caption = get_alternative_caption(image_np, model, text_embeddings, testing_df["caption"])
+                    regenerated_radiology_report = generate_radiology_report(f"Write Complete Radiology Report for this with clinical info, subjective, Assessment, Finding, Impressions, Conclusion and more in proper order : {alternative_caption}")
+
+                    regenerated_radiology_report_with_personal_info = f"Patient Name: {first_name} {last_name}\nAge: {age}\nGender: {gender}\n\n{regenerated_radiology_report}"
+
+                    st.header("Regenerated Radiology Report")
+                    st.write(regenerated_radiology_report_with_personal_info)
+                    st.markdown(download_link(save_as_docx(regenerated_radiology_report_with_personal_info, "regenerated_radiology_report.docx"), "regenerated_radiology_report.docx", "Download Regenerated Report as DOCX"), unsafe_allow_html=True)

gitignore.txt

@@ -0,0 +1,7 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*~
+.env
+

main.py

@@ -0,0 +1,335 @@
+from torch import nn
+from tqdm.autonotebook import tqdm
+from transformers import AutoTokenizer, AutoModel
+from transformers import DistilBertModel, DistilBertConfig, DistilBertTokenizer
+import albumentations as A
+import cv2
+import timm
+import torch
+import torch.nn.functional as F
+
+device = torch.device("cpu")
+
+class CFG:
+    debug = False
+    image_path = '/content/content/new_images_v5'
+    captions_path = '/content/content/all_data/new_caption.csv'
+    batch_size = 12
+    num_workers = 2
+    head_lr = 1e-3
+    image_encoder_lr = 1e-4
+    text_encoder_lr = 1e-5
+    weight_decay = 1e-3
+    patience = 1
+    factor = 0.8
+    epochs = 2
+    saved_model_clinical = '/content/content/new_weights.pt'
+    trained_model = 'clinical_bert_weights.pt'
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    model_name = 'resnet50'
+    image_embedding = 2048
+    text_encoder_model = "distilbert-base-uncased"
+    clinical_encoder_model = "emilyalsentzer/Bio_ClinicalBERT"
+    text_embedding = 768
+    text_tokenizer = "distilbert-base-uncased"
+    max_length = 200
+
+    pretrained = True  # for both image encoder and text encoder
+    trainable = True  # for both image encoder and text encoder
+    temperature = 1.0
+
+    # image size
+    size = 224
+
+    # for projection head; used for both image and text encoders
+    num_projection_layers = 1
+    projection_dim = 256
+    dropout = 0.1
+
+
+def build_loaders(dataframe, tokenizer, mode):
+    transforms = get_transforms(mode=mode)
+    dataset = CLIPDataset(
+        dataframe["image"].values,
+        dataframe["caption"].values,
+        tokenizer=tokenizer,
+        transforms=transforms,
+    )
+
+    dataloader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=CFG.batch_size,
+        num_workers=CFG.num_workers,
+        shuffle=True if mode == "train" else False,
+    )
+    return dataloader
+
+
+
+class AvgMeter:
+    def __init__(self, name="Metric"):
+        self.name = name
+        self.reset()
+
+    def reset(self):
+        self.avg, self.sum, self.count = [0] * 3
+
+    def update(self, val, count=1):
+        self.count += count
+        self.sum += val * count
+        self.avg = self.sum / self.count
+
+    def __repr__(self):
+        text = f"{self.name}: {self.avg:.4f}"
+        return text
+
+def get_lr(optimizer):
+    for param_group in optimizer.param_groups:
+        return param_group["lr"]
+
+
+# Custom dataset object. Will tokenize text and apply transforms to images before yielding them.
+
+class CLIPDataset(torch.utils.data.Dataset):
+    def __init__(self, image_filenames, captions, tokenizer, transforms):
+        """
+        image_filenames and cpations must have the same length; so, if there are
+        multiple captions for each image, the image_filenames must have repetitive
+        file names
+        """
+
+        self.image_filenames = image_filenames
+        self.captions = list(captions)
+        self.skippedImgCount = 0
+        self.encoded_captions = tokenizer(
+            list(captions), padding=True, truncation=True, max_length=CFG.max_length
+        )
+        self.transforms = transforms
+
+    def __getitem__(self, idx):
+        item = {
+            key: torch.tensor(values[idx])
+            for key, values in self.encoded_captions.items()
+        }
+
+        image = cv2.imread(f"{CFG.image_path}/{self.image_filenames[idx]}")
+        if image is None:
+            # Skip the current example and move to the next one
+            self.skippedImgCount += 1
+            return self.__getitem__((idx + 1) % len(self))
+
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        image = self.transforms(image=image)['image']
+        item['image'] = torch.tensor(image).permute(2, 0, 1).float()
+        item['caption'] = self.captions[idx]
+
+        return item
+
+    def __len__(self):
+        return len(self.captions)
+
+
+def get_transforms(mode="train"):
+    if mode == "train":
+        return A.Compose(
+            [
+                A.Resize(CFG.size, CFG.size, always_apply=True),
+                A.Normalize(max_pixel_value=255.0, always_apply=True),
+            ]
+        )
+    else:
+        return A.Compose(
+            [
+                A.Resize(CFG.size, CFG.size, always_apply=True),
+                A.Normalize(max_pixel_value=255.0, always_apply=True),
+            ]
+        )
+
+
+class ImageEncoder(nn.Module):
+    """
+    Encode images to a fixed size vector
+    """
+
+    def __init__(
+        self, model_name=CFG.model_name, pretrained=CFG.pretrained, trainable=CFG.trainable
+    ):
+        super().__init__()
+        self.model = timm.create_model(
+            model_name, pretrained, num_classes=0, global_pool="avg"
+        )
+        for p in self.model.parameters():
+            p.requires_grad = trainable
+
+    def forward(self, x):
+        return self.model(x)
+
+class TextEncoder(nn.Module):
+    def __init__(self, model_name=CFG.text_encoder_model, pretrained=CFG.pretrained, trainable=CFG.trainable):
+        super().__init__()
+        if pretrained:
+            # self.model = DistilBertModel.from_pretrained(model_name)
+
+            # Use Bio-ClinicalBERT
+            self.model = AutoModel.from_pretrained(CFG.clinical_encoder_model)
+
+        else:
+            self.model = DistilBertModel(config=DistilBertConfig())
+
+        for p in self.model.parameters():
+            p.requires_grad = trainable
+
+        # we are using the CLS token hidden representation as the sentence's embedding
+        self.target_token_idx = 0
+
+    def forward(self, input_ids, attention_mask):
+        output = self.model(input_ids=input_ids, attention_mask=attention_mask)
+        last_hidden_state = output.last_hidden_state
+        return last_hidden_state[:, self.target_token_idx, :]
+
+
+# Get both image and text encodings into a same size matrix
+class ProjectionHead(nn.Module):
+    def __init__(
+            self,
+            embedding_dim,
+            projection_dim=CFG.projection_dim,
+            dropout=CFG.dropout
+    ):
+        super().__init__()
+        self.projection = nn.Linear(embedding_dim, projection_dim)
+        self.gelu = nn.GELU()
+        self.fc = nn.Linear(projection_dim, projection_dim)
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(projection_dim)
+
+    def forward(self, x):
+        projected = self.projection(x)
+        x = self.gelu(projected)
+        x = self.fc(x)
+        x = self.dropout(x)
+        x = x + projected
+        x = self.layer_norm(x)
+        return x
+
+
+class CLIPModel(nn.Module):
+    def __init__(
+        self,
+        temperature=CFG.temperature,
+        image_embedding=CFG.image_embedding,
+        text_embedding=CFG.text_embedding,
+    ):
+        super().__init__()
+        self.image_encoder = ImageEncoder()
+        self.text_encoder = TextEncoder()
+        self.image_projection = ProjectionHead(embedding_dim=image_embedding)
+        self.text_projection = ProjectionHead(embedding_dim=text_embedding)
+        self.temperature = temperature
+
+    def forward(self, batch):
+        # Getting Image and Text Features
+        image_features = self.image_encoder(batch["image"])
+        text_features = self.text_encoder(
+            input_ids=batch["input_ids"], attention_mask=batch["attention_mask"]
+        )
+        # Getting Image and Text Embeddings (with same dimension)
+        image_embeddings = self.image_projection(image_features)
+        text_embeddings = self.text_projection(text_features)
+
+        # Calculating the Loss
+        logits = (text_embeddings @ image_embeddings.T) / self.temperature
+        images_similarity = image_embeddings @ image_embeddings.T
+        texts_similarity = text_embeddings @ text_embeddings.T
+        targets = F.softmax(
+            (images_similarity + texts_similarity) / 2 * self.temperature, dim=-1
+        )
+        texts_loss = cross_entropy(logits, targets, reduction='none')
+        images_loss = cross_entropy(logits.T, targets.T, reduction='none')
+        loss =  (images_loss + texts_loss) / 2.0 # shape: (batch_size)
+        return loss.mean()
+def cross_entropy(preds, targets, reduction='none'):
+    log_softmax = nn.LogSoftmax(dim=-1)
+    loss = (-targets * log_softmax(preds)).sum(1)
+    if reduction == "none":
+        return loss
+    elif reduction == "mean":
+        return loss.mean()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# INFERENCE CODE
+def get_image_embeddings(image):
+    # preprocess the image
+    if image is None:
+        print("Image not found!")
+        return None
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    image = get_transforms("valid")(image=image)['image']
+    image = image.reshape(3, 224, 224)
+    model = CLIPModel().to(device)
+    model.load_state_dict(torch.load('weights.pt', map_location=device))
+    model.eval()
+
+    with torch.no_grad():
+        image_tensor = torch.from_numpy(image)
+        image_features = model.image_encoder(image_tensor.unsqueeze(0).to(device))
+        image_embeddings = model.image_projection(image_features)
+        image_embeddings = F.normalize(image_embeddings, p=2, dim=-1)
+
+    return image_embeddings
+
+
+def predict_caption(image, model, text_embeddings, captions, n=2):
+    # get the image embeddings
+    image_embeddings = get_image_embeddings(image)
+    if image_embeddings is None:
+        return None
+
+    # normalize the embeddings
+    image_embeddings_n = F.normalize(image_embeddings, p=2, dim=-1)
+    text_embeddings_n = F.normalize(text_embeddings, p=2, dim=-1)
+    # calculate the dot product of image and text embeddings
+    dot_similarity = image_embeddings_n @ text_embeddings_n.T
+
+    # get the top n matches
+    values, indices = torch.topk(dot_similarity.squeeze(0), n)
+    indices = indices.cpu().numpy().tolist()
+    matches = [captions[idx] for idx in indices]
+
+    return matches
+
+def get_text_embeddings(valid_df):
+    tokenizer = AutoTokenizer.from_pretrained(CFG.clinical_encoder_model)
+    valid_loader = build_loaders(valid_df, tokenizer, mode="valid")
+
+    model = CLIPModel().to(device)
+    model.load_state_dict(torch.load("weights.pt", map_location=device))
+    model.eval()
+
+    valid_text_embeddings = []
+    with torch.no_grad():
+        for batch in tqdm(valid_loader):
+            text_features = model.text_encoder(
+                input_ids=batch["input_ids"].to(device), attention_mask=batch["attention_mask"].to(device)
+            )
+            text_embeddings = model.text_projection(text_features)
+            valid_text_embeddings.append(text_embeddings)
+
+    return model, torch.cat(valid_text_embeddings)

requirements.txt

@@ -0,0 +1,14 @@
+torch
+opencv-python==4.5.4.60
+transformers
+albumentations
+timm
+tqdm
+openai
+streamlit==0.84.0
+pandas
+torch
+Pillow
+numpy
+huggingface-hub
+python-docx

saved_text_embeddings.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2da707e595ccab006a2159d26d55469c7f015ea4a4dbc645154972fa96a17cc5
+size 7538453

weights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7be01315d364ccdce0cc260f8b43c7381e3629e670017e5aaa9bcc6ca172eb34
+size 531111517