update handler

example_client.py

@@ -0,0 +1,116 @@
+import requests
+import json
+import os
+import argparse
+
+def query_endpoint(endpoint_url, api_token=None, prompt="Hello, how are you?", 
+                  system_message="You are a helpful assistant.", 
+                  max_tokens=256, temperature=0.7, 
+                  format_type="openai"):
+    """
+    Query the Phi-4 Mini model at the specified HuggingFace Inference Endpoint.
+    
+    Args:
+        endpoint_url: The URL of your HuggingFace Inference Endpoint
+        api_token: Your HuggingFace API token (if needed)
+        prompt: The user message to send to the model
+        system_message: The system message to include
+        max_tokens: Maximum number of tokens to generate
+        temperature: Temperature for generation (0.0 to 1.0)
+        format_type: Type of request format to use:
+                    "openai" - Standard OpenAI format
+                    "hf_wrapped" - HuggingFace format with OpenAI format wrapped in "inputs"
+                    "simple" - Simple text input in "inputs" field
+    
+    Returns:
+        The response from the model
+    """
+    # Prepare headers
+    headers = {
+        "Content-Type": "application/json"
+    }
+    
+    if api_token:
+        headers["Authorization"] = f"Bearer {api_token}"
+    
+    # Prepare the request payload based on format_type
+    if format_type == "openai":
+        # Standard OpenAI format
+        payload = {
+            "messages": [
+                {"role": "system", "content": system_message},
+                {"role": "user", "content": prompt}
+            ],
+            "max_tokens": max_tokens,
+            "temperature": temperature
+        }
+    elif format_type == "hf_wrapped":
+        # HuggingFace wrapped format
+        payload = {
+            "inputs": {
+                "messages": [
+                    {"role": "system", "content": system_message},
+                    {"role": "user", "content": prompt}
+                ],
+                "max_tokens": max_tokens,
+                "temperature": temperature
+            }
+        }
+    elif format_type == "simple":
+        # Simple text input
+        payload = {
+            "inputs": prompt
+        }
+    else:
+        raise ValueError(f"Invalid format type: {format_type}")
+    
+    # Make the request
+    try:
+        print(f"Request payload: {json.dumps(payload, indent=2)}")
+        response = requests.post(endpoint_url, headers=headers, data=json.dumps(payload))
+        response.raise_for_status()  # Raise an exception for HTTP errors
+        
+        # Parse and return the response
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        print(f"Error making request: {e}")
+        if hasattr(e, 'response') and e.response:
+            print(f"Response content: {e.response.text}")
+        return None
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Query a Phi-4 Mini HuggingFace Inference Endpoint")
+    parser.add_argument("--url", type=str, required=True, help="The endpoint URL")
+    parser.add_argument("--token", type=str, default=os.environ.get("HF_API_TOKEN"), help="HuggingFace API token")
+    parser.add_argument("--prompt", type=str, default="Explain quantum computing in simple terms.", help="User prompt")
+    parser.add_argument("--system", type=str, default="You are a helpful assistant.", help="System message")
+    parser.add_argument("--max_tokens", type=int, default=256, help="Maximum tokens to generate")
+    parser.add_argument("--temperature", type=float, default=0.7, help="Temperature (0.0 to 1.0)")
+    parser.add_argument("--format", type=str, default="openai", 
+                        choices=["openai", "hf_wrapped", "simple"],
+                        help="Format to use for the request")
+    
+    args = parser.parse_args()
+    
+    print(f"Querying endpoint: {args.url}")
+    print(f"Prompt: {args.prompt}")
+    print(f"Format: {args.format}")
+    
+    response = query_endpoint(
+        args.url, 
+        args.token, 
+        args.prompt, 
+        args.system, 
+        args.max_tokens, 
+        args.temperature,
+        args.format
+    )
+    
+    if response:
+        print("\nResponse:")
+        if "choices" in response and len(response["choices"]) > 0:
+            print(response["choices"][0]["message"]["content"])
+        else:
+            print(json.dumps(response, indent=2))
+    else:
+        print("Failed to get a valid response") 

handler.py

@@ -34,10 +34,44 @@ class EndpointHandler:
         print("Model loaded successfully")
         
     def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Handle inference request in OpenAI-like format"""
+        """Handle inference request in OpenAI-like format or HuggingFace Inference API format"""
         try:
+            # Debugging: Print the received data
+            print(f"Received data: {json.dumps(data, indent=2)}")
+            
+            # Handle HuggingFace Inference API format
+            if "inputs" in data:
+                # Extract data from inputs key
+                if isinstance(data["inputs"], dict):
+                    # If inputs contains a dictionary, extract it
+                    input_data = data["inputs"]
+                elif isinstance(data["inputs"], str):
+                    # If inputs is a string, create a simple message
+                    input_data = {
+                        "messages": [
+                            {"role": "user", "content": data["inputs"]}
+                        ]
+                    }
+                else:
+                    print(f"Unexpected inputs format: {type(data['inputs'])}")
+                    # Try to convert to string if possible
+                    try:
+                        input_data = {
+                            "messages": [
+                                {"role": "user", "content": str(data["inputs"])}
+                            ]
+                        }
+                    except:
+                        raise ValueError(f"Unsupported inputs format: {type(data['inputs'])}")
+            else:
+                # Assume direct OpenAI format
+                input_data = data
+            
+            # Debugging: Print the parsed input data
+            print(f"Parsed input data: {json.dumps(input_data, indent=2)}")
+            
             # Parse input data
-            inputs = self._parse_input(data)
+            inputs = self._parse_input(input_data)
             
             # Generate response
             outputs = self._generate(inputs)
@@ -45,6 +79,9 @@ class EndpointHandler:
             # Format response in OpenAI-like format
             return self._format_response(outputs, inputs)
         except Exception as e:
+            print(f"Error during processing: {str(e)}")
+            import traceback
+            traceback.print_exc()
             return {
                 "error": {
                     "message": str(e),
@@ -58,6 +95,7 @@ class EndpointHandler:
         # Extract messages
         messages = data.get("messages", [])
         if not messages:
+            print(f"No messages found in data: {json.dumps(data, indent=2)}")
             raise ValueError("No messages provided")
         
         # Convert messages to prompt

test_hf_endpoint.py

@@ -0,0 +1,63 @@
+import requests
+import json
+import argparse
+
+def test_hf_endpoint(endpoint_url, api_token=None):
+    """Test the HuggingFace Inference Endpoint with different formats"""
+
+    # Format 1: Wrapped OpenAI format
+    payload1 = {
+        "inputs": {
+            "messages": [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "Explain quantum computing in simple terms."}
+            ],
+            "max_tokens": 256,
+            "temperature": 0.7
+        }
+    }
+
+    # Format 2: Simple string
+    payload2 = {
+        "inputs": "Tell me about AI."
+    }
+
+    # Prepare headers
+    headers = {
+        "Content-Type": "application/json",
+    }
+    
+    if api_token:
+        headers["Authorization"] = f"Bearer {api_token}"
+
+    # Test Format 1
+    print("Testing Format 1: Wrapped OpenAI format...")
+    print(f"Payload: {json.dumps(payload1, indent=2)}")
+    
+    try:
+        response = requests.post(endpoint_url, headers=headers, json=payload1)
+        print(f"Status Code: {response.status_code}")
+        print(f"Response: {json.dumps(response.json(), indent=2)}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+
+    print("\n" + "-"*50 + "\n")
+
+    # Test Format 2
+    print("Testing Format 2: Simple string...")
+    print(f"Payload: {json.dumps(payload2, indent=2)}")
+    
+    try:
+        response = requests.post(endpoint_url, headers=headers, json=payload2)
+        print(f"Status Code: {response.status_code}")
+        print(f"Response: {json.dumps(response.json(), indent=2)}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Test HuggingFace Inference Endpoint")
+    parser.add_argument("--url", type=str, required=True, help="Endpoint URL")
+    parser.add_argument("--token", type=str, help="API token (if needed)")
+    
+    args = parser.parse_args()
+    test_hf_endpoint(args.url, args.token)