add code

vouchervision/LLM_chatGPT_3_5.py

@@ -223,6 +223,8 @@ def structured_output_parser(is_azure, MODEL, llm, prompt_template, logger, prom
         return None
 
 def structured_output_parser_for_function_calls_fail(is_azure, MODEL, failed_response, logger, llm, prompt_version, is_helper=False, try_ind=0):
+    if try_ind == 0:
+        original_failed_response = failed_response
     if try_ind > 5:
         return None
 
@@ -272,13 +274,17 @@ def structured_output_parser_for_function_calls_fail(is_azure, MODEL, failed_res
     except json.decoder.JSONDecodeError as e:
         try_ind += 1
         error_message = str(e)
-        redo_content = f'The error messsage is: {error_message}\nThe broken JSON object is: {output.content}'
-        logger.info(f'[Failed JSON Object]\n{output.content}')
-        refined_response = structured_output_parser_for_function_calls_fail(is_azure, MODEL, redo_content, logger, llm, prompt_version, is_helper, try_ind)
+        redo_content = f'The error messsage is: {error_message}\nThe broken JSON object is: {original_failed_response}'  # Use original_failed_response here
+        logger.info(f'[Failed JSON Object]\n{original_failed_response}')  # And here
+        refined_response = structured_output_parser_for_function_calls_fail(
+            is_azure, MODEL, redo_content, logger, llm, prompt_version, is_helper, try_ind, original_failed_response
+        )
     except:
         try_ind += 1
-        logger.info(f'[Failed JSON Object]\n{output.content}')
-        refined_response = structured_output_parser_for_function_calls_fail(is_azure, MODEL, output.content, logger, llm, prompt_version, is_helper, try_ind)
+        logger.info(f'[Failed JSON Object]\n{original_failed_response}')  # And here
+        refined_response = structured_output_parser_for_function_calls_fail(
+            is_azure, MODEL, original_failed_response, logger, llm, prompt_version, is_helper, try_ind, original_failed_response
+        )
 
     return refined_response
 

vouchervision/general_utils.py

@@ -34,11 +34,22 @@ def get_cfg_from_full_path(path_cfg):
         cfg = yaml.full_load(ymlfile)
     return cfg
 
-def num_tokens_from_string(string: str, encoding_name: str) -> int:
-    encoding = tiktoken.get_encoding(encoding_name)
-    if isinstance(string, dict):
-        string = json.dumps(string, sort_keys=False)
-    num_tokens = len(encoding.encode(string))
+def num_tokens_from_string(string, encoding_name):
+    try:
+        # Ensure the encoding is obtained correctly.
+        encoding = tiktoken.get_encoding(encoding_name)
+        
+        # Convert dictionary to string if it is not already a string
+        if isinstance(string, dict):
+            string = json.dumps(string, ensure_ascii=False)
+        
+        # Encode the string and return the number of tokens.
+        num_tokens = len(encoding.encode(string))
+    except Exception as e:
+        # If there's any error, log it and return 0.
+        print(f"An error occurred: {e}")
+        num_tokens = 0
+
     return num_tokens
 
 def add_to_expense_report(dir_home, data):