context:
im having issues quering code snippets based on natural language description using coderbert model
Files:
embeddings.py
``# Code for generating embeddings for your codebase

# 1. Install Required Libraries
# Ensure you have installed the 'sentence-transformers' library
# If not, you can install it using:
# pip install sentence-transformers

# 2. Import Necessary Libraries
import json
import numpy as np
from sentence_transformers import SentenceTransformer, models
import torch

# 3. Load Your Codebase
# Replace 'codebase.json' with the path to your codebase JSON file
with open('/home/dheena/Downloads/Intiliee/output/output_code_data.json', 'r') as f:
    codebase = json.load(f)

# Extract code snippets and metadata
code_snippets = []
metadata = []

for file in codebase:
    file_name = file.get('file_name', '')
    snippets = file.get('code_snippets', {})
    functions = snippets.get('functions', [])
    classes = snippets.get('classes', [])

    for func in functions:
        code_snippets.append(func)
        metadata.append({'type': 'function', 'file_name': file_name})

    for cls in classes:
        code_snippets.append(cls)
        metadata.append({'type': 'class', 'file_name': file_name})

print(f"Total code snippets loaded: {len(code_snippets)}")
from transformers import AutoTokenizer, AutoModel

# 4. Load the Fine-Tuned Code Search Model
model_name = 'microsoft/codebert-base'  # Fine-tuned for code search
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# 5. Generate Embeddings for the Code Snippets
def get_code_embedding(code_snippet):
    inputs = tokenizer(
        code_snippet,
        return_tensors='pt',
        truncation=True,
        padding='max_length',
        max_length=256
    )
    with torch.no_grad():
        outputs = model(**inputs)
        embedding = outputs.last_hidden_state[:, 0, :]
        return embedding.squeeze().numpy()

print("Generating embeddings for code snippets...")
embeddings = []
for snippet in code_snippets:
    embedding = get_code_embedding(snippet)
    embeddings.append(embedding)

embeddings = np.array(embeddings)
np.save('/home/dheena/Downloads/Intiliee/output/embeddings.npy', embeddings)

# Save code snippets and metadata as JSON files
with open('/home/dheena/Downloads/Intiliee/output/code_snippets.json', 'w') as f:
    json.dump(code_snippets, f)

with open('/home/dheena/Downloads/Intiliee/output/metadata.json', 'w') as f:
    json.dump(metadata, f)

print("Embeddings and data have been saved successfully.")``

query.py
``import faiss
import json
import numpy as np
import torch
from sentence_transformers import SentenceTransformer, models
from transformers import AutoTokenizer, AutoModel

# Paths to your saved files
FAISS_INDEX_PATH = '/home/dheena/Downloads/Intiliee/output/code_embeddings.index'
EMBEDDINGS_PATH = '/home/dheena/Downloads/Intiliee/output/embeddings.npy'
CODE_SNIPPETS_PATH = '/home/dheena/Downloads/Intiliee/output/code_snippets.json'
METADATA_PATH = '/home/dheena/Downloads/Intiliee/output/metadata.json'

# Load the Faiss index
index = faiss.read_index(FAISS_INDEX_PATH)
print("Faiss index loaded successfully.")

# Load embeddings
embeddings = np.load(EMBEDDINGS_PATH)
print(f"Embeddings loaded successfully with shape: {embeddings.shape}")

# Load code snippets
with open(CODE_SNIPPETS_PATH, 'r') as f:
    code_snippets = json.load(f)
print(f"Loaded {len(code_snippets)} code snippets.")

# Load metadata
with open(METADATA_PATH, 'r') as f:
    metadata = json.load(f)
print(f"Loaded metadata for {len(metadata)} code snippets.")

# Load the tokenizer and fine-tuned model
model_name = 'microsoft/codebert-base'  # Fine-tuned for code search
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

print("Model loaded successfully.")

def get_query_embedding(query_text):
    inputs = tokenizer(
        query_text,
        return_tensors='pt',
        truncation=True,
        padding='max_length',
        max_length=256
    )
    with torch.no_grad():
        outputs = model(**inputs)
        embedding = outputs.last_hidden_state[:, 0, :]
        return embedding.squeeze().numpy().reshape(1, -1)

# Normalize embeddings if necessary
def normalize_embedding(embedding):
    norm = np.linalg.norm(embedding, axis=1, keepdims=True)
    return embedding / norm

# Modify search function to include normalization
def search_code_snippets(user_query, index, code_snippets, metadata, top_k=5):
    query_embedding = get_query_embedding(user_query)
    query_embedding = normalize_embedding(query_embedding)
    distances, indices = index.search(query_embedding, k=top_k)
    results = []
    for idx in indices[0]:
        snippet = code_snippets[idx]
        info = metadata[idx]
        similarity = distances[0][np.where(indices[0] == idx)[0][0]]
        results.append({
            'snippet': snippet,
            'metadata': info,
            'similarity': similarity
        })
    return results


# Sample user query
user_query = "What is the purpose of the deletePauseOption() method?"

# Search for similar code snippets
results = search_code_snippets(user_query, index, code_snippets, metadata, top_k=5)

# Display the results
print("\nTop matching code snippets:")
for i, result in enumerate(results):
    print(f"\nResult {i+1}:")
    print(f"File: {result['metadata']['file_name']}")
    print(f"Type: {result['metadata']['type']}")
    print("Code Snippet:")
    print(result['snippet'])``
indexing.py
``from random import randint
import faiss
import numpy as np
import json

# Load embeddings from the .npy file
embeddings = np.load('/home/dheena/Downloads/Intiliee/output/embeddings.npy')

# Load code snippets and metadata
with open('/home/dheena/Downloads/Intiliee/output/code_snippets.json', 'r') as f:
    code_snippets = json.load(f)

with open('/home/dheena/Downloads/Intiliee/output/metadata.json', 'r') as f:
    metadata = json.load(f)

print(f"Embeddings shape: {embeddings.shape}")
print(f"Number of code snippets: {len(code_snippets)}")
print(f"Number of metadata entries: {len(metadata)}")

# Normalize the embeddings
# Normalize the embeddings
embeddings = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)

if embeddings.dtype != 'float32':
    embeddings = embeddings.astype('float32')

# Create the index for cosine similarity
embedding_dimension = embeddings.shape[1]
index = faiss.IndexFlatIP(embedding_dimension)

# Add embeddings to the index
index.add(embeddings)

print(f"Number of vectors in the index: {index.ntotal}")

faiss.write_index(index, '/home/dheena/Downloads/Intiliee/output/code_embeddings.index')
print("Index saved successfully.")

# Let's take the embedding of the first code snippet as a test query
test_embedding = embeddings[0].reshape(1, -1)


# Number of nearest neighbors to retrieve
k = 5

# Perform the search
distances, indices = index.search(test_embedding, k)

print("Indices of nearest neighbors:", indices)
print("Distances to nearest neighbors:", distances)``
output:
"[Intiliee]  cd /home/dheena/Downloads/Intiliee ; /usr/bin/env /bin/python3 /home/dheena/.vscode/ex
tensions/ms-python.debugpy-2024.14.0-linux-x64/bundled/libs/debugpy/adapter/../../debugpy/launcher
 46145 -- /home/dheena/Downloads/Intiliee/query.py 
Faiss index loaded successfully.
Embeddings loaded successfully with shape: (1027, 768)
Loaded 1027 code snippets.
Loaded metadata for 1027 code snippets.
Model loaded successfully.

Top matching code snippets:

Result 1:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/test/java/com/kapture/ticketconfigurationmicroservice/TicketConfigurationMicroserviceApplicationTests.java
Type: function
Code Snippet:
@Test
        void contextLoads() {
        }

Result 2:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/test/java/com/kapture/ticketconfigurationmicroservice/TicketConfigurationMicroserviceApplicationTests.java
Type: class
Code Snippet:
@SpringBootTest
class TicketConfigurationMicroserviceApplicationTests {

        @Test
        void contextLoads() {
        }

}

Result 3:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/main/java/com/kapture/ticketconfigurationmicroservice/ticket/service/WebHookDataPrivilege.java
Type: function
Code Snippet:
public String getCategory() {
        return category;
    }

Result 4:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/main/java/com/kapture/ticketconfigurationmicroservice/ticket/service/WebHookDataPrivilege.java
Type: function
Code Snippet:
public String getKey() {
        return key;
    }

Result 5:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/main/java/com/kapture/ticketconfigurationmicroservice/pojo/ExcelTicketFolderMapping.java
Type: function
Code Snippet:
public String getDataType() {
        return dataType;
    }
[Intiliee]  cd /home/dheena/Downloads/Intiliee ; /usr/bin/env /bin/python3 /home/dheena/.vscode/ex
tensions/ms-python.debugpy-2024.14.0-linux-x64/bundled/libs/debugpy/adapter/../../debugpy/launcher
 59443 -- /home/dheena/Downloads/Intiliee/query.py 
Faiss index loaded successfully.
Embeddings loaded successfully with shape: (1027, 768)
Loaded 1027 code snippets.
Loaded metadata for 1027 code snippets.
Model loaded successfully.

Top matching code snippets:

Result 1:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/test/java/com/kapture/ticketconfigurationmicroservice/TicketConfigurationMicroserviceApplicationTests.java
Type: function
Code Snippet:
@Test
        void contextLoads() {
        }

Result 2:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/test/java/com/kapture/ticketconfigurationmicroservice/TicketConfigurationMicroserviceApplicationTests.java
Type: class
Code Snippet:
@SpringBootTest
class TicketConfigurationMicroserviceApplicationTests {

        @Test
        void contextLoads() {
        }

}

Result 3:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/main/java/com/kapture/ticketconfigurationmicroservice/ticket/service/WebHookDataPrivilege.java
Type: function
Code Snippet:
public String getCategory() {
        return category;
    }

Result 4:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/main/java/com/kapture/ticketconfigurationmicroservice/ticket/service/WebHookDataPrivilege.java
Type: function
Code Snippet:
public String getKey() {
        return key;
    }

Result 5:
File: /home/dheena/workspace/microservices/ticket-configuration-microservice/src/main/java/com/kapture/ticketconfigurationmicroservice/pojo/ExcelTicketFolderMapping.java
Type: function
Code Snippet:
public String getDataType() {
        return dataType;
    }" irrevelant snippets