Update question_generator.py

question_generator.py

@@ -23,7 +23,48 @@ model = "mistral-large-latest"
 # Initialize Mistral client
 client = MistralClient(api_key=api_key)
 
-# ... (previous functions remain the same)
+def load_csv_data(file_path: str) -> List[Dict[str, str]]:
+    """Load data from a CSV file."""
+    logging.info(f"Loading data from {file_path}...")
+    try:
+        with open(file_path, 'r', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+            data = list(reader)
+        logging.info(f"Loaded {len(data)} rows from {file_path}")
+        return data
+    except FileNotFoundError:
+        logging.error(f"File not found: {file_path}")
+        raise
+    except csv.Error as e:
+        logging.error(f"Error reading CSV file {file_path}: {e}")
+        raise
+
+# Load data from both CSV files
+try:
+    detailed_cases = load_csv_data('processed_medical_history.csv')
+    infectious_diseases = load_csv_data('infectious_diseases.csv')
+except Exception as e:
+    logging.error(f"Failed to load CSV data: {e}")
+    raise
+
+def hash_question(question: str) -> str:
+    """Generate a hash for a question to check for duplicates."""
+    return hashlib.md5(question.encode()).hexdigest()
+
+def load_generated_questions() -> set:
+    """Load previously generated question hashes from a file."""
+    try:
+        with open('generated_questions.txt', 'r') as f:
+            return set(line.strip() for line in f)
+    except FileNotFoundError:
+        return set()
+
+def save_generated_question(question_hash: str):
+    """Save a newly generated question hash to the file."""
+    with open('generated_questions.txt', 'a') as f:
+        f.write(question_hash + '\n')
+
+generated_questions = load_generated_questions()
 
 def generate_microbiology_question() -> Dict[str, str]:
     """Generate a microbiology question."""