Update app.py

app.py

@@ -1,7 +1,7 @@
 import logging
 import textwrap
 from typing import Literal, Optional
-
+import os
 import gradio as gr
 import outlines
 import pandas as pd
@@ -17,6 +17,8 @@ from transformers import (
     BitsAndBytesConfig,
 )
 
+
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -70,10 +72,28 @@ PROMPT_TEMPLATE = textwrap.dedent("""
 </Answer>
 Score:""").strip()
 
+if is_huggingface_space():
+    DEVICE_MAP = "cpu"
+    QUANTIZATION_BITS = None
+else:
+    DEVICE_MAP = "auto"
+    QUANTIZATION_BITS = 4  # or whatever you prefer for local deployment
+
+def is_huggingface_space():
+    return os.environ.get('SPACE_ID') is not None
+
+
 
 def get_outlines_model(
-    model_id: str, device_map: str = "auto", quantization_bits: Optional[int] = 4
+    model_id: str, device_map: str = "cpu", quantization_bits: Optional[int] = None
 ):
+     # Skip quantization on CPU
+    if device_map == "cpu":
+        quantization_config = None
+    else:
+        # Your existing quantization logic
+        pass
+        
     if quantization_bits == 4:
         quantization_config = BitsAndBytesConfig(
             load_in_4bit=True,
@@ -120,7 +140,7 @@ def format_prompt(story: str, question: str, grading_scheme: str, answer: str) -
     return full_prompt
 
 
-@spaces.GPU
+# @spaces.GPU
 def label_single_response_with_model(model_id, story, question, criteria, response):
     prompt = format_prompt(story, question, criteria, response)
     logger.info(f"Prompt: {prompt}")
@@ -143,7 +163,7 @@ def label_single_response_with_model(model_id, story, question, criteria, respon
         return result.score
 
 
-@spaces.GPU
+# @spaces.GPU
 def label_multi_responses_with_model(
     model_id, story, question, criteria, response_file
 ):