feat: add batching

feat: add smaller model

app.py

@@ -6,7 +6,7 @@ from distilabel.llms import LlamaCppLLM
 from distilabel.steps.tasks.argillalabeller import ArgillaLabeller
 
 file_path = os.path.join(os.path.dirname(__file__), "Qwen2-5-0.5B-Instruct-f16.gguf")
-download_url = "https://huggingface.co/gaianet/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-f16.gguf?download=true"
+download_url = "https://huggingface.co/gaianet/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q8_0.gguf?download=true"
 
 
 if not os.path.exists(file_path):
@@ -66,14 +66,15 @@ def process_records_gradio(records, example_records, fields, question):
         runtime_parameters = {"fields": fields, "question": question}
         if example_records:
             runtime_parameters["example_records"] = example_records
-        print(runtime_parameters)
+
         task.set_runtime_parameters(runtime_parameters)
 
         results = []
-        for record in records:
-            output = next(task.process(inputs=[{"records": record}]))
-            if output[0]["suggestions"]:
-                results.append(output[0]["suggestions"].serialize())
+        output = task.process(inputs=[{"records": record} for record in records])
+        for _ in range(len(records)):
+            entry = next(output)[0]
+            if entry["suggestions"]:
+                results.append(entry["suggestions"].serialize())
 
         return json.dumps({"results": results}, indent=2)
     except Exception as e: