hi you and I have be refactoring and getting tests to pass on a key component of my CLI tool in a previous session unfortunately we got disconnected. We just recently got all the unit tests passing and now we were about to look at getting our integration test refactored to reflect our new updated classes. I will provide you the file code and unit tests for context. Then after you get a chance to review and understand whats going on and calibrate your understanding by explaining the current state of the code, I will provide the integration test in a follow up query! sound good? heres the code:

what do you think about all the advanced techniques used in our confidence scorer?

# confidence_scorer.py

from typing import Dict, Any, Tuple, List, Set
from rapidfuzz import fuzz
from nltk.corpus import wordnet
import nltk
import re
import logging
from functools import lru_cache
import json
import numpy as np # Import Numpy for array handling
import warnings

# Ensure NLTK WordNet data is downloaded
try:
nltk.data.find('corpora/wordnet')
except LookupError:
nltk.download('wordnet', quiet=True)

# ----------------- Constants -----------------

# Confidence thresholds
HIGH_CONFIDENCE_THRESHOLD: float = 0.7
MEDIUM_CONFIDENCE_THRESHOLD: float = 0.55
LOW_CONFIDENCE_THRESHOLD: float = 0.3

# Relevant keywords for scope determination
RELEVANT_KEYWORDS: Set[str] = {
'import', 'comment', 'print', 'file', 'global',
'function', 'class', 'method', 'update', 'modify',
'change', 'compute', 'calculate'
}

# Synonym mapping using WordNet


def initialize_synonym_map() -> Dict[str, Set[str]]:
"""
Initializes a synonym map for relevant keywords using WordNet.

Returns:
Dict[str, Set[str]]: A dictionary mapping each keyword to a set of its synonyms.
"""
synonym_map: Dict[str, Set[str]] = {}
for keyword in RELEVANT_KEYWORDS:
synonyms = {lemma.name().lower().replace('_', ' ') for syn in wordnet.synsets(keyword) for lemma in syn.lemmas()}
synonym_map[keyword] = synonyms
return synonym_map


SYNONYM_MAP = initialize_synonym_map()

# ----------------- Logging Configuration -----------------

logging.basicConfig(
level=logging.DEBUG, # Adjust as needed (DEBUG, INFO, WARNING, etc.)
format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(name)

# ----------------- ConfidenceScorer Class -------------------


class ConfidenceScorer:
"""
Base class for calculating confidence scores for code edit operations.

Attributes:
efc (Dict[str, Any]): Enhanced File Context containing code structure.
query (str): The edit instruction or query.
model (SentenceTransformer): Pre-trained model for generating embeddings.
query_embedding (np.ndarray): Embedding vector for the query.
"""

def init(self, efc: Dict[str, Any], query: str, model_name: str = 'all-MiniLM-L6-v2'):
"""
Initializes the ConfidenceScorer with the provided Enhanced File Context and query.

Args:
efc (Dict[str, Any]): Enhanced File Context.
query (str): The edit instruction or query.
model_name (str, optional): Pre-trained model name. Defaults to 'all-MiniLM-L6-v2'.
"""
self.efc: Dict[str, Any] = efc
self.query: str = query
self.model = self._initialize_model(model_name)
self.query_embedding: np.ndarray = self.get_embedding(self.query)

@staticmethod
def _initialize_model(model_name: str):
"""
Initializes the SentenceTransformer model.

Args:
model_name (str): Name of the pre-trained model.

Returns:
SentenceTransformer: The initialized model.
"""
try:
from sentence_transformers import SentenceTransformer
return SentenceTransformer(model_name)
except ImportError:
logger.error("Please install 'sentence-transformers' package.")
raise

@lru_cache(maxsize=128)
def get_embedding(self, text: str) -> np.ndarray:
"""
Generates an embedding for the given text using the pre-trained model.

Args:
text (str): Text to be embedded.

Returns:
np.ndarray: The embedding vector.
"""
with warnings.catch_warnings():
warnings.simplefilter("ignore", FutureWarning)
return self.model.encode(text, convert_to_tensor=False)

def calculate_confidence(self) -> float:
"""
Calculates the overall confidence score based on semantic similarity.

Returns:
float: Confidence score normalized between 0 and 1.
"""
code_block_names = [block['name'] for block in self.efc.get('custom_tokens', [])]
code_block_contents = [block['code'] for block in self.efc.get('custom_tokens', [])]

name_embeddings = [self.get_embedding(name) for name in code_block_names]
content_embeddings = [self.get_embedding(content) for content in code_block_contents]

best_name_score = self._get_best_similarity(self.query_embedding, name_embeddings)
best_content_score = self._get_best_similarity(self.query_embedding, content_embeddings)

# Weighted confidence: name and content contribute equally
confidence = (0.5 * best_name_score) + (0.5 * best_content_score)
logger.debug(f"Calculated confidence - Name: {best_name_score}, Content: {best_content_score}, Combined: {confidence}")
return confidence

@staticmethod
def _get_best_similarity(query_embedding: np.ndarray, embeddings: List[np.ndarray]) -> float:
"""
Computes the highest cosine similarity score between the query embedding and a list of embeddings.

Args:
query_embedding (np.ndarray): Embedding vector for the query.
embeddings (List[np.ndarray]): List of embedding vectors to compare against.

Returns:
float: The highest similarity score between 0 and 1.
"""
if not embeddings:
return 0.0
similarities = [ConfidenceScorer._cosine_similarity(query_embedding, emb) for emb in embeddings]
best_similarity = max(similarities)
return best_similarity

@staticmethod
def _cosine_similarity(vec1: np.ndarray, vec2: np.ndarray) -> float:
"""
Computes the cosine similarity between two vectors.

Args:
vec1 (np.ndarray): First vector.
vec2 (np.ndarray): Second vector.

Returns:
float: Cosine similarity between 0 and 1.
"""
if vec1.size == 0 or vec2.size == 0:
return 0.0
dot_product = np.dot(vec1, vec2)
magnitude1 = np.linalg.norm(vec1)
magnitude2 = np.linalg.norm(vec2)
if magnitude1 == 0 or magnitude2 == 0:
return 0.0
return dot_product / (magnitude1 * magnitude2)

def assess_complexity(self) -> float:
"""
Assesses the complexity of the edit based on the length and keyword richness of the query.

Returns:
float: Complexity score normalized between 0 and 1.
"""
query_length = len(self.query.split())
keyword_count = len(set(self.query.lower().split()).intersection(RELEVANT_KEYWORDS))
complexity = (query_length / 20) + (keyword_count / len(RELEVANT_KEYWORDS))
normalized_complexity = min(complexity / 2, 1.0) # Normalize to 0-1
logger.debug(f"Assessed complexity: {normalized_complexity} (length: {query_length}, keywords: {keyword_count})")
return normalized_complexity

def determine_scope(self) -> Tuple[str, float]:
"""
Determines the appropriate scope for the edit operation based on confidence and complexity.

Returns:
Tuple[str, float]: The determined scope and confidence score.
"""
confidence: float = self.calculate_confidence()
complexity: float = self.assess_complexity()

has_relevant_keyword: bool = self._has_relevant_keyword()

if not has_relevant_keyword:
scope = "file-level edit"
else:
# Prioritize function scope if function keywords are present
function_keywords = {'function', 'def', 'method', 'update', 'modify', 'change', 'compute', 'calculate'}
query_words = set(re.findall(r'\w+', self.query.lower()))
is_function_related = bool(query_words.intersection(function_keywords))

if is_function_related and confidence > HIGH_CONFIDENCE_THRESHOLD and complexity < 0.3:
scope = "function"
elif confidence > MEDIUM_CONFIDENCE_THRESHOLD and complexity < 0.6:
scope = "class"
else:
scope = "file-level edit"

logger.debug(f"Determine Scope - Confidence: {confidence}, Complexity: {complexity}, Scope: {scope}")
return scope, confidence

def _has_relevant_keyword(self) -> bool:
"""
Checks if the query contains any relevant keywords or their synonyms.

Returns:
bool: True if relevant keywords are present, False otherwise.
"""
query_words = set(re.findall(r'\w+', self.query.lower()))
for keyword, synonyms in SYNONYM_MAP.items():
if query_words.intersection(synonyms.union({keyword})):
logger.debug(f"Relevant keyword detected: {keyword}")
return True
return False


# ----------------- EnhancedConfidenceScorer Class -----------------

class EnhancedConfidenceScorer(ConfidenceScorer):
"""
Advanced confidence scoring system for code edit operations.

Extends ConfidenceScorer with multi-dimensional confidence scoring and enhanced keyword recognition.
"""

def init(self, efc: Dict[str, Any], query: str, model_name: str = 'all-MiniLM-L6-v2'):
super().init(efc, query, model_name)

def calculate_confidence(self) -> Dict[str, float]:
"""
Calculates multi-dimensional confidence scores:
- overall: Combined confidence score
- name: Confidence in identifying the correct code block
- content: Confidence in understanding the required changes
- context: Confidence in the surrounding code context

Returns:
Dict[str, float]: Dictionary containing confidence scores.
"""
name_similarity = self._calculate_name_confidence()
content_similarity = self._calculate_content_confidence()
context_confidence = self._calculate_context_confidence()

# Weighting factors
NAME_WEIGHT: float = 0.35
CONTENT_WEIGHT: float = 0.5
CONTEXT_WEIGHT: float = 0.15

base_confidence: float = (
name_similarity * NAME_WEIGHT +
content_similarity * CONTENT_WEIGHT +
context_confidence * CONTEXT_WEIGHT
)

# Apply boosting based on operation type
boosted_confidence: float = self._boost_confidence(base_confidence)

# Apply graceful degradation for low confidence
confidence_scores: Dict[str, float] = {
"overall": round(float(boosted_confidence), 2),
"name": round(float(name_similarity), 2),
"content": round(float(content_similarity), 2),
"context": round(float(context_confidence), 2)
}
confidence_scores = self._apply_graceful_degradation(confidence_scores)

logger.debug(f"Calculated confidence scores: {json.dumps(confidence_scores, indent=2)}")
return confidence_scores

def determine_scope(self) -> Tuple[str, Dict[str, float]]:
"""
Determines the edit scope based on multi-dimensional confidence scores.

Returns:
Tuple[str, Dict[str, float]]: The determined scope and confidence scores.
"""
confidence_scores: Dict[str, float] = self.calculate_confidence()
complexity: float = self.assess_complexity()

logger.debug(f"Instruction: {self.query}")
logger.debug(f"Confidence scores: {confidence_scores}")
logger.debug(f"Complexity: {complexity}")

if not self.query.strip():
confidence_scores = {
"overall": 0.0,
"name": 0.0,
"content": 0.0,
"context": 0.5 # Retain default context confidence
}
logger.debug("Empty query detected. Defaulting to 'file-level edit' with zero confidence.")
return "file-level edit", confidence_scores

if self._is_file_level_operation():
logger.debug("Scope determined: File-level edit (file-level operation detected)")
return "file-level edit", confidence_scores

# Prioritize class-level operations first
if self._is_class_level_operation():
if confidence_scores['overall'] > 0.6 and complexity < 0.6:
scope = "class"
else:
scope = "file-level edit"
logger.debug(f"Scope determined: {scope} (Class-level operation)")
return scope, confidence_scores

# Then check for function-level operations
if self._is_function_level_operation():
if confidence_scores['overall'] > 0.7 and complexity < 0.3:
scope = "function"
else:
scope = "file-level edit"
logger.debug(f"Scope determined: {scope} (Function-level operation)")
return scope, confidence_scores

# Existing logic for file-level or ambiguous scopes
if not self._has_relevant_keyword():
scope = "file-level edit"
logger.debug("Scope determined: File-level edit (irrelevant or vague instruction)")
elif confidence_scores['overall'] > HIGH_CONFIDENCE_THRESHOLD:
scope = "function"
logger.debug("Scope determined: Function (high confidence)")
elif confidence_scores['overall'] > MEDIUM_CONFIDENCE_THRESHOLD:
scope = "class"
logger.debug("Scope determined: Class (medium confidence)")
else:
scope = "file-level edit"
logger.debug("Scope determined: File-level edit (default case)")
return scope, confidence_scores

def _calculate_name_confidence(self) -> float:
"""
Calculates confidence in identifying the correct code block based on names.

Returns:
float: Name confidence score between 0 and 1.
"""
code_block_names = [block['name'] for block in self.efc.get('custom_tokens', [])]
name_similarity = self._get_best_similarity(self.query_embedding, [self.get_embedding(name) for name in code_block_names])
logger.debug(f"Name similarity score: {name_similarity}")
return name_similarity

def _calculate_content_confidence(self) -> float:
"""
Calculates confidence in understanding the required changes based on code content.

Returns:
float: Content confidence score between 0 and 1.
"""
code_block_contents = [block['code'] for block in self.efc.get('custom_tokens', [])]
content_similarity = self._get_best_similarity(self.query_embedding, [self.get_embedding(content) for content in code_block_contents])
logger.debug(f"Content similarity score: {content_similarity}")
return content_similarity

def _calculate_context_confidence(self) -> float:
"""
Calculates confidence in the surrounding code context.

Returns:
float: Context confidence score between 0 and 1.
"""
# Example: Higher confidence if the query mentions keywords related to context
contextual_keywords: Set[str] = {'syntax', 'structure', 'format', 'refactor'}
query_tags = set(re.findall(r'\w+', self.query.lower()))
common_keywords = query_tags.intersection(contextual_keywords)
context_confidence = 1.0 if common_keywords else 0.5
logger.debug(f"Context confidence score: {context_confidence}")
return context_confidence

def _boost_confidence(self, base_confidence: float) -> float:
"""
Boosts the confidence score based on the type of operation detected.

Args:
base_confidence (float): The initial confidence score.

Returns:
float: The boosted confidence score, capped at 0.95.
"""
if self._is_file_level_operation():
boosted = min(base_confidence * 1.2, 0.95)
logger.debug(f"Boosted confidence for file-level operation: {boosted}")
return boosted
elif self._is_function_level_operation():
boosted = min(base_confidence * 1.2, 0.95) # Increased boost multiplier
logger.debug(f"Boosted confidence for function-level operation: {boosted}")
return boosted
return base_confidence

def _apply_graceful_degradation(self, confidence_scores: Dict[str, float]) -> Dict[str, float]:
"""
Applies graceful degradation to confidence scores if overall confidence is low.

Args:
confidence_scores (Dict[str, float]): The current confidence scores.

Returns:
Dict[str, float]: The adjusted confidence scores.
"""
if confidence_scores['overall'] < LOW_CONFIDENCE_THRESHOLD:
degraded_scores = {
k: min(v * 0.8, 0.4) if k != 'context' else 0.5
for k, v in confidence_scores.items()
}
logger.debug(f"Applied graceful degradation: {degraded_scores}")
return degraded_scores
return confidence_scores

def _is_file_level_operation(self) -> bool:
"""
Determines if the operation is at the file level based on keywords.

Returns:
bool: True if file-level operation, False otherwise.
"""
file_level_keywords = {'import', 'comment', 'print', 'file', 'global'}
query_words = set(re.findall(r'\w+', self.query.lower()))
is_file_level = bool(query_words.intersection(file_level_keywords))
logger.debug(f"Is file-level operation: {is_file_level}")
return is_file_level

def _is_function_level_operation(self) -> bool:
"""
Determines if the operation is at the function level based on keywords.

Returns:
bool: True if function-level operation, False otherwise.
"""
function_keywords = {'function', 'def', 'method', 'update', 'modify'}
query_words = set(re.findall(r'\w+', self.query.lower()))
is_function_level = bool(query_words.intersection(function_keywords))
logger.debug(f"Is function-level operation: {is_function_level}")
return is_function_level

def _is_class_level_operation(self) -> bool:
"""
Determines if the operation is at the class level based on keywords.

Returns:
bool: True if class-level operation, False otherwise.
"""
class_keywords = {'class'}
query_words = set(re.findall(r'\w+', self.query.lower()))
is_class_level = bool(query_words.intersection(class_keywords))
logger.debug(f"Is class-level operation: {is_class_level}")
return is_class_level

# test_confidence_scorer.py

import pytest
from efc.efc_utils import ConfidenceScorer, generate_hybrid_efc, EnhancedConfidenceScorer
from smart_edit.smart_edit import find_best_matching_block, determine_update_scope
from typing import Dict, Any


# ----------------- Fixtures -----------------

@pytest.fixture
def sample_efc() -> Dict[str, Any]:
"""
Provides a sample Enhanced File Context (EFC) for testing.
"""
return {
'custom_tokens': [
{
'type': 'import_statement',
'name': 'datetime_import',
'code': 'import datetime'
},
{
'type': 'function_definition',
'name': 'calculate_total',
'code': 'def calculate_total(items):\n return sum(item.price for item in items)'
},
{
'type': 'class_definition',
'name': 'ShoppingCart',
'code': 'class ShoppingCart:\n def init(self):\n self.items = []\n\n def add_item(self, item):\n self.items.append(item)'
}
]
}
# ----------------- Tests for EnhancedConfidenceScorer -----------------

def test_enhanced_confidence_scorer_calculate_confidence(sample_efc):
"""
Test that EnhancedConfidenceScorer.calculate_confidence() returns a dictionary with correct keys and values.
"""
query = "Update calculate_total function to handle discounts."
scorer = EnhancedConfidenceScorer(sample_efc, query)
confidence_scores = scorer.calculate_confidence()

# Ensure all values are standard Python floats
assert isinstance(confidence_scores, dict), "Confidence scores should be a dictionary."
for key in ['overall', 'name', 'content', 'context']:
assert key in confidence_scores, f"Confidence scores should contain '{key}'."
assert isinstance(confidence_scores[key], float), f"{key} score should be a standard float."
assert 0.0 <= confidence_scores[key] <= 1.0, f"{key} score should be between 0 and 1."


def test_enhanced_confidence_scorer_determine_scope_function(sample_efc):
"""
Test that EnhancedConfidenceScorer.determine_scope() correctly identifies a function-level edit with high confidence.
"""
query = "Update calculate_total function to handle discounts."
scorer = EnhancedConfidenceScorer(sample_efc, query)
scope, confidence_scores = scorer.determine_scope()

assert scope == "function", "Scope should be identified as 'function'."
assert confidence_scores['overall'] > 0.7, "Overall confidence should be high for function-level edits."
assert confidence_scores['name'] > 0.6, "Name confidence should be high."
assert confidence_scores['content'] > 0.55, "Content confidence should be high."
assert confidence_scores['context'] == 0.5, "Context confidence should be defaulted to 0.5."


def test_enhanced_confidence_scorer_determine_scope_class(sample_efc):
"""
Test that EnhancedConfidenceScorer.determine_scope() correctly identifies a class-level edit with sufficient confidence.
"""
query = "Modify ShoppingCart class to add remove_item method."
scorer = EnhancedConfidenceScorer(sample_efc, query)
scope, confidence_scores = scorer.determine_scope()

assert scope == "class", "Scope should be identified as 'class'."
assert confidence_scores['overall'] > 0.6, "Overall confidence should be sufficient for class-level edits."
assert confidence_scores['name'] > 0.55, "Name confidence should be sufficiently close to 0.6."
assert confidence_scores['content'] > 0.6, "Content confidence should be sufficient."
assert confidence_scores['context'] == 0.5, "Context confidence should be defaulted to 0.5."


def test_enhanced_confidence_scorer_determine_scope_file_level(sample_efc):
"""
Test that EnhancedConfidenceScorer.determine_scope() correctly identifies a file-level edit.
"""
query = "Add import statement for datetime."
scorer = EnhancedConfidenceScorer(sample_efc, query)
scope, confidence_scores = scorer.determine_scope()

assert scope == "file-level edit", "Scope should be identified as 'file-level edit'."
assert confidence_scores['overall'] > 0.5, "Overall confidence should be adequate for file-level edits."
assert confidence_scores['name'] > 0.5, "Name confidence should be adequate."
assert confidence_scores['content'] > 0.5, "Content confidence should be adequate."
assert confidence_scores['context'] == 0.5, "Context confidence should be defaulted to 0.5."


def test_enhanced_confidence_scorer_determine_scope_irrelevant(sample_efc):
"""
Test that EnhancedConfidenceScorer.determine_scope() defaults to file-level edit for irrelevant queries.
"""
query = "Refactor codebase."
scorer = EnhancedConfidenceScorer(sample_efc, query)
scope, confidence_scores = scorer.determine_scope()

assert scope == "file-level edit", "Scope should default to 'file-level edit' for irrelevant queries."
assert confidence_scores['overall'] < 0.6, "Overall confidence should be low for irrelevant queries."
assert confidence_scores['name'] < 0.6, "Name confidence should be low."
assert confidence_scores['content'] < 0.6, "Content confidence should be low."
assert confidence_scores['context'] == 0.5, "Context confidence should be defaulted to 0.5."


def test_enhanced_confidence_scorer_edge_case_empty_query(sample_efc):
"""
Test EnhancedConfidenceScorer with an empty query to ensure graceful handling.
"""
query = ""
scorer = EnhancedConfidenceScorer(sample_efc, query)
scope, confidence_scores = scorer.determine_scope()

assert scope == "file-level edit", "Empty query should default to 'file-level edit'."
assert confidence_scores['overall'] == 0.0, "Overall confidence should be zero for empty queries."
assert confidence_scores['name'] == 0.0, "Name confidence should be zero."
assert confidence_scores['content'] == 0.0, "Content confidence should be zero."
assert confidence_scores['context'] == 0.5, "Context confidence should remain defaulted to 0.5."