feat: improve tool calling & add tools to qwen 2.5 72b (#1615)

* fix: clean up tool calling

more inline with latest changes in transformers lib and adds tool calling to qwen 2.5 72b

* fix: add more field name options

* fix: make tool calls a bit more robust

chart/env/prod.yaml

@@ -50,7 +50,6 @@ envVars:
     [
       {
         "name": "Qwen/Qwen2.5-72B-Instruct",
-        "tokenizer": "Qwen/Qwen2.5-72B-Instruct",
         "description": "The latest Qwen open model with improved role-playing, long text generation and structured data understanding.",
         "modelUrl": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct",
         "websiteUrl": "https://qwenlm.github.io/blog/qwen2.5/",
@@ -62,6 +61,7 @@ envVars:
           "truncate": 28672,
           "max_new_tokens": 3072
         },
+        "tools": true,
         "promptExamples": [
           {
             "title": "Write an email from bullet list",
@@ -80,7 +80,6 @@ envVars:
       {
         "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
         "id": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-        "tokenizer": "nsarrazin/llama3.1-tokenizer",
         "description": "Ideal for everyday use. A fast and extremely capable model matching closed source models' capabilities.",
         "modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct",
         "websiteUrl": "https://llama.meta.com/",
@@ -110,7 +109,6 @@ envVars:
       },
       {
         "name": "CohereForAI/c4ai-command-r-plus-08-2024",
-        "tokenizer": "nsarrazin/c4ai-command-r-v01-tokenizer",
         "description": "Cohere's largest language model, optimized for conversational interaction and tool use. Now with the 2024 update!",
         "modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024",
         "websiteUrl": "https://docs.cohere.com/docs/command-r-plus",
@@ -350,10 +348,6 @@ envVars:
       },
       {
         "name": "llhf/Meta-Llama-3.1-8B-Instruct",
-        "tokenizer": {
-          "tokenizerUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/resolve/main/tokenizer.json",
-          "tokenizerConfigUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/raw/main/tokenizer_config.json"
-        },
         "parameters": {
           "temperature": 0.6,
           "stop": ["<|endoftext|>", "<|eot_id|>"]

package-lock.json

@@ -10,10 +10,10 @@
 			"dependencies": {
 				"@aws-sdk/credential-providers": "^3.592.0",
 				"@cliqz/adblocker-playwright": "^1.27.2",
-				"@gradio/client": "^1.1.1",
+				"@gradio/client": "^1.8.0",
 				"@huggingface/hub": "^0.5.1",
-				"@huggingface/inference": "^2.7.0",
-				"@huggingface/transformers": "^3.0.0-alpha.6",
+				"@huggingface/inference": "^2.8.1",
+				"@huggingface/transformers": "^3.1.1",
 				"@iconify-json/bi": "^1.1.21",
 				"@playwright/browser-chromium": "^1.43.1",
 				"@resvg/resvg-js": "^2.6.2",
@@ -1929,9 +1929,10 @@
 			}
 		},
 		"node_modules/@gradio/client": {
-			"version": "1.5.2",
-			"resolved": "https://registry.npmjs.org/@gradio/client/-/client-1.5.2.tgz",
-			"integrity": "sha512-PJbt8n3ROxDXrR55D8oPYPIIkdmG5ksyljtgNM6Kr/FgTMTwGqiiYDdoyUIGwQSBu6g+jan2zbHoniDZePw3pg==",
+			"version": "1.8.0",
+			"resolved": "https://registry.npmjs.org/@gradio/client/-/client-1.8.0.tgz",
+			"integrity": "sha512-0Nxi+AuEL/ICdnZ6TSxRif1yS2A0HtJnggtWhN/8sqwFxx5UAGN29/vnJONcVfe3g9nctDJCvik3uDHkzQHFGw==",
+			"license": "ISC",
 			"dependencies": {
 				"@types/eventsource": "^1.1.15",
 				"bufferutil": "^4.0.7",
@@ -1959,37 +1960,41 @@
 			}
 		},
 		"node_modules/@huggingface/inference": {
-			"version": "2.8.0",
-			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.8.0.tgz",
-			"integrity": "sha512-Ti681P1qckcCAqgzmL53jBnluPuZGelmMIuXNjgAwC5+RIjF4S0SDQu6oy44ZTwekwNp2ETaZ2sXsOk+45aC4w==",
+			"version": "2.8.1",
+			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.8.1.tgz",
+			"integrity": "sha512-EfsNtY9OR6JCNaUa5bZu2mrs48iqeTz0Gutwf+fU0Kypx33xFQB4DKMhp8u4Ee6qVbLbNWvTHuWwlppLQl4p4Q==",
+			"license": "MIT",
 			"dependencies": {
-				"@huggingface/tasks": "^0.11.2"
+				"@huggingface/tasks": "^0.12.9"
 			},
 			"engines": {
 				"node": ">=18"
 			}
 		},
 		"node_modules/@huggingface/jinja": {
-			"version": "0.3.0",
-			"resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.0.tgz",
-			"integrity": "sha512-GLJzso0M07ZncFkrJMIXVU4os6GFbPocD4g8fMQPMGJubf48FtGOsUORH2rtFdXPIPelz8SLBMn8ZRmOTwXm9Q==",
+			"version": "0.3.2",
+			"resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.2.tgz",
+			"integrity": "sha512-F2FvuIc+w1blGsaqJI/OErRbWH6bVJDCBI8Rm5D86yZ2wlwrGERsfIaru7XUv9eYC3DMP3ixDRRtF0h6d8AZcQ==",
+			"license": "MIT",
 			"engines": {
 				"node": ">=18"
 			}
 		},
 		"node_modules/@huggingface/tasks": {
-			"version": "0.11.13",
-			"resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.13.tgz",
-			"integrity": "sha512-TqFEyFtKYAYwDg9h4XQMzoSxN2NMpwSnerPBx7Y4RbM1nHLM+CTXAUHcDY+hydcA5CoDDaBGzkHw+mttY3AmFQ=="
+			"version": "0.12.30",
+			"resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.12.30.tgz",
+			"integrity": "sha512-A1ITdxbEzx9L8wKR8pF7swyrTLxWNDFIGDLUWInxvks2ruQ8PLRBZe8r0EcjC3CDdtlj9jV1V4cgV35K/iy3GQ==",
+			"license": "MIT"
 		},
 		"node_modules/@huggingface/transformers": {
-			"version": "3.0.0-alpha.14",
-			"resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.0.0-alpha.14.tgz",
-			"integrity": "sha512-yRTKOP+xcZqHtlr7WiMWrV+E+XvqfiVLF4wZAlD1ozf0O5LPXKdkd8bfxFogd5d1kDxPlEdCLCJ3l8ZTNfqRyA==",
+			"version": "3.1.1",
+			"resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.1.1.tgz",
+			"integrity": "sha512-/OpCiSKIowo5w5rJAOH3pgZKvpT6DOfDYw9br9Fp8w3qm4oyxc6dOhrxdRLVrmqLbE8rp5dKCePUW34ZBsinsg==",
+			"license": "Apache-2.0",
 			"dependencies": {
-				"@huggingface/jinja": "^0.3.0",
-				"onnxruntime-node": "1.19.0",
-				"onnxruntime-web": "1.20.0-dev.20240827-1d059b8702",
+				"@huggingface/jinja": "^0.3.2",
+				"onnxruntime-node": "1.20.1",
+				"onnxruntime-web": "1.20.1",
 				"sharp": "^0.33.5"
 			}
 		},
@@ -2639,6 +2644,7 @@
 			"version": "4.0.1",
 			"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
 			"integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+			"license": "ISC",
 			"dependencies": {
 				"minipass": "^7.0.4"
 			},
@@ -2808,27 +2814,32 @@
 		"node_modules/@protobufjs/aspromise": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
-			"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
+			"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/base64": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
-			"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
+			"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/codegen": {
 			"version": "2.0.4",
 			"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
-			"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
+			"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/eventemitter": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
-			"integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
+			"integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/fetch": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
 			"integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+			"license": "BSD-3-Clause",
 			"dependencies": {
 				"@protobufjs/aspromise": "^1.1.1",
 				"@protobufjs/inquire": "^1.1.0"
@@ -2837,27 +2848,32 @@
 		"node_modules/@protobufjs/float": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
-			"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
+			"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/inquire": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
-			"integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
+			"integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/path": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
-			"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
+			"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/pool": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
-			"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
+			"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@protobufjs/utf8": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
-			"integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
+			"integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+			"license": "BSD-3-Clause"
 		},
 		"node_modules/@remusao/guess-url-type": {
 			"version": "1.3.0",
@@ -5512,6 +5528,7 @@
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
 			"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+			"license": "BlueOak-1.0.0",
 			"engines": {
 				"node": ">=18"
 			}
@@ -7103,7 +7120,8 @@
 		"node_modules/flatbuffers": {
 			"version": "1.12.0",
 			"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz",
-			"integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ=="
+			"integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==",
+			"license": "SEE LICENSE IN LICENSE.txt"
 		},
 		"node_modules/flatted": {
 			"version": "3.3.1",
@@ -7489,7 +7507,8 @@
 		"node_modules/guid-typescript": {
 			"version": "1.0.9",
 			"resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
-			"integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="
+			"integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
+			"license": "ISC"
 		},
 		"node_modules/handlebars": {
 			"version": "4.7.8",
@@ -8749,7 +8768,8 @@
 		"node_modules/long": {
 			"version": "5.2.3",
 			"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
-			"integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q=="
+			"integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==",
+			"license": "Apache-2.0"
 		},
 		"node_modules/loupe": {
 			"version": "3.1.2",
@@ -8988,6 +9008,7 @@
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz",
 			"integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==",
+			"license": "MIT",
 			"dependencies": {
 				"minipass": "^7.0.4",
 				"rimraf": "^5.0.5"
@@ -9000,6 +9021,7 @@
 			"version": "5.0.10",
 			"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz",
 			"integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==",
+			"license": "ISC",
 			"dependencies": {
 				"glob": "^10.3.7"
 			},
@@ -9677,43 +9699,41 @@
 			}
 		},
 		"node_modules/onnxruntime-common": {
-			"version": "1.19.0",
-			"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.19.0.tgz",
-			"integrity": "sha512-Oo16UIJ/xLOtZDVGcL4bL8EP8MiNFztyBmR3pB14D+cl/UCpOgHHzEk0MADSmYXQ0FgyEegPXtOFcJqhq1YRsw=="
+			"version": "1.20.1",
+			"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.1.tgz",
+			"integrity": "sha512-YiU0s0IzYYC+gWvqD1HzLc46Du1sXpSiwzKb63PACIJr6LfL27VsXSXQvt68EzD3V0D5Bc0vyJTjmMxp0ylQiw==",
+			"license": "MIT"
 		},
 		"node_modules/onnxruntime-node": {
-			"version": "1.19.0",
-			"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.19.0.tgz",
-			"integrity": "sha512-JivYcrZ9H9YPpHnP+5rTVTJjzuD+ZB0TsB0//e2La0ViYNG0hmTvnwFgmRoes6F7E1PyVMpyRftmcl9pnIWsnw==",
+			"version": "1.20.1",
+			"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.20.1.tgz",
+			"integrity": "sha512-di/I4HDXRw+FLgq+TyHmQEDd3cEp9iFFZm0r4uJ1Wd7b/WE1VXtKWo8yemex347c6GNF/3Pv86ZfPhIWxORr0w==",
 			"hasInstallScript": true,
+			"license": "MIT",
 			"os": [
 				"win32",
 				"darwin",
 				"linux"
 			],
 			"dependencies": {
-				"onnxruntime-common": "1.19.0",
+				"onnxruntime-common": "1.20.1",
 				"tar": "^7.0.1"
 			}
 		},
 		"node_modules/onnxruntime-web": {
-			"version": "1.20.0-dev.20240827-1d059b8702",
-			"resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.20.0-dev.20240827-1d059b8702.tgz",
-			"integrity": "sha512-v1t3TgnOmyzA1lGcUiqp6kZDv7QxO+yJI00vHp8lyjgBrnxkLY+6fqFZpBfm81AH5sRaCdG+HsOmb5Y37TIAyw==",
+			"version": "1.20.1",
+			"resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.20.1.tgz",
+			"integrity": "sha512-TePF6XVpLL1rWVMIl5Y9ACBQcyCNFThZON/jgElNd9Txb73CIEGlklhYR3UEr1cp5r0rbGI6nDwwrs79g7WjoA==",
+			"license": "MIT",
 			"dependencies": {
 				"flatbuffers": "^1.12.0",
 				"guid-typescript": "^1.0.9",
 				"long": "^5.2.3",
-				"onnxruntime-common": "1.20.0-dev.20240827-5d54dc1462",
+				"onnxruntime-common": "1.20.1",
 				"platform": "^1.3.6",
 				"protobufjs": "^7.2.4"
 			}
 		},
-		"node_modules/onnxruntime-web/node_modules/onnxruntime-common": {
-			"version": "1.20.0-dev.20240827-5d54dc1462",
-			"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.0-dev.20240827-5d54dc1462.tgz",
-			"integrity": "sha512-oR+xPRD64OI+w9nRLXQi9rEXYZ5W9BhqVi688sUTreU9J6pK182JYblmHjvapCg+Tta6MbkAsr3T1NZHM3tB1g=="
-		},
 		"node_modules/openai": {
 			"version": "4.58.1",
 			"resolved": "https://registry.npmjs.org/openai/-/openai-4.58.1.tgz",
@@ -10245,7 +10265,8 @@
 		"node_modules/platform": {
 			"version": "1.3.6",
 			"resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
-			"integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="
+			"integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+			"license": "MIT"
 		},
 		"node_modules/playwright": {
 			"version": "1.47.0",
@@ -10631,6 +10652,7 @@
 			"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
 			"integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==",
 			"hasInstallScript": true,
+			"license": "BSD-3-Clause",
 			"dependencies": {
 				"@protobufjs/aspromise": "^1.1.2",
 				"@protobufjs/base64": "^1.1.2",
@@ -12124,6 +12146,7 @@
 			"version": "7.4.3",
 			"resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
 			"integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
+			"license": "ISC",
 			"dependencies": {
 				"@isaacs/fs-minipass": "^4.0.0",
 				"chownr": "^3.0.0",
@@ -12152,6 +12175,7 @@
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
 			"integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
+			"license": "MIT",
 			"bin": {
 				"mkdirp": "dist/cjs/src/bin.js"
 			},
@@ -12166,6 +12190,7 @@
 			"version": "5.0.0",
 			"resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
 			"integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+			"license": "BlueOak-1.0.0",
 			"engines": {
 				"node": ">=18"
 			}

package.json

@@ -63,10 +63,10 @@
 	"dependencies": {
 		"@aws-sdk/credential-providers": "^3.592.0",
 		"@cliqz/adblocker-playwright": "^1.27.2",
-		"@gradio/client": "^1.1.1",
+		"@gradio/client": "^1.8.0",
 		"@huggingface/hub": "^0.5.1",
-		"@huggingface/inference": "^2.7.0",
-		"@huggingface/transformers": "^3.0.0-alpha.6",
+		"@huggingface/inference": "^2.8.1",
+		"@huggingface/transformers": "^3.1.1",
 		"@iconify-json/bi": "^1.1.21",
 		"@playwright/browser-chromium": "^1.43.1",
 		"@resvg/resvg-js": "^2.6.2",

src/lib/server/models.ts

@@ -129,7 +129,11 @@ async function getChatPromptRender(
 		toolResults,
 		continueMessage,
 	}: ChatTemplateInput) => {
-		let formattedMessages: { role: string; content: string }[] = messages.map((message) => ({
+		let formattedMessages: {
+			role: string;
+			content: string;
+			tool_calls?: { id: string; tool_call_id: string; output: string }[];
+		}[] = messages.map((message) => ({
 			content: message.content,
 			role: message.from,
 		}));
@@ -166,7 +170,7 @@ async function getChatPromptRender(
 			if (isHuggingChat && id.startsWith("CohereForAI")) {
 				formattedMessages = [
 					{
-						role: m.systemRoleSupported ? "system" : "user",
+						role: "user",
 						content:
 							"\n\n<results>\n" +
 							toolResults
@@ -226,20 +230,6 @@ async function getChatPromptRender(
 			tools = [];
 		}
 
-		const chatTemplate = tools?.length ? "tool_use" : undefined;
-
-		const documents = (toolResults ?? []).flatMap((result) => {
-			if (result.status === ToolResultStatus.Error) {
-				return [{ title: `Tool "${result.call.name}" error`, text: "\n" + result.message }];
-			}
-			return result.outputs.flatMap((output) =>
-				Object.entries(output).map(([title, text]) => ({
-					title: `Tool "${result.call.name}" ${title}`,
-					text: "\n" + text,
-				}))
-			);
-		});
-
 		const mappedTools =
 			tools?.map((tool) => {
 				const inputs: Record<
@@ -271,9 +261,7 @@ async function getChatPromptRender(
 		const output = tokenizer.apply_chat_template(formattedMessages, {
 			tokenize: false,
 			add_generation_prompt: !continueMessage,
-			chat_template: chatTemplate,
-			tools: mappedTools,
-			documents,
+			tools: mappedTools.length ? mappedTools : undefined,
 		});
 
 		if (typeof output !== "string") {
@@ -282,7 +270,6 @@ async function getChatPromptRender(
 
 		return output;
 	};
-
 	return renderTemplate;
 }
 

src/lib/server/textGeneration/tools.ts

@@ -184,11 +184,12 @@ export async function* runTools(
 		? [...formattedMessages.slice(0, -1), fileMsg, ...formattedMessages.slice(-1)]
 		: messages;
 
+	let rawText = "";
 	// do the function calling bits here
 	for await (const output of await endpoint({
 		messages: formattedMessages,
 		preprompt,
-		generateSettings: assistant?.generateSettings,
+		generateSettings: { temperature: 0.1, ...assistant?.generateSettings },
 		tools: tools.map((tool) => ({
 			...tool,
 			inputs: tool.inputs.map((input) => ({
@@ -204,24 +205,44 @@ export async function* runTools(
 			continue;
 		}
 
+		if (output.token.text) {
+			rawText += output.token.text;
+		}
+
+		// if we dont see a tool call in the first 25 chars, something is going wrong and we abort
+		if (rawText.length > 25 && !(rawText.includes("```json") || rawText.includes("{"))) {
+			return [];
+		}
+
+		// if we see a directly_answer tool call, we skip the rest
+		if (
+			rawText.includes("directly_answer") ||
+			rawText.includes("directlyAnswer") ||
+			rawText.includes("directly-answer")
+		) {
+			return [];
+		}
+
 		// look for a code blocks of ```json and parse them
 		// if they're valid json, add them to the calls array
 		if (output.generated_text) {
 			try {
 				const rawCalls = await extractJson(output.generated_text);
 				const newCalls = rawCalls
-					.filter(isExternalToolCall)
 					.map((call) => externalToToolCall(call, tools))
 					.filter((call) => call !== undefined) as ToolCall[];
 
 				calls.push(...newCalls);
 			} catch (e) {
-				logger.error(e, "Error while parsing tool calls, please retry");
+				logger.warn(
+					{ rawCall: output.generated_text, error: extractJson },
+					"Error while parsing tool calls"
+				);
 				// error parsing the calls
 				yield {
 					type: MessageUpdateType.Status,
 					status: MessageUpdateStatus.Error,
-					message: "Error while parsing tool calls, please retry",
+					message: "Error while parsing tool calls.",
 				};
 			}
 		}
@@ -239,36 +260,33 @@ export async function* runTools(
 	return toolResults.filter((result): result is ToolResult => result !== undefined);
 }
 
-const externalToolCall = z.object({
-	tool_name: z.string(),
-	parameters: z.record(z.any()),
-});
-
-type ExternalToolCall = z.infer<typeof externalToolCall>;
-
-function isExternalToolCall(call: unknown): call is ExternalToolCall {
-	return externalToolCall.safeParse(call).success;
-}
+function externalToToolCall(call: unknown, tools: Tool[]): ToolCall | undefined {
+	// Early return if invalid input
+	if (!isValidCallObject(call)) {
+		return undefined;
+	}
 
-function externalToToolCall(call: ExternalToolCall, tools: Tool[]): ToolCall | undefined {
-	// Convert - to _ since some models insist on using _ instead of -
-	const tool = tools.find((tool) => toolHasName(call.tool_name, tool));
+	const parsedCall = parseExternalCall(call);
+	if (!parsedCall) return undefined;
 
+	const tool = tools.find((tool) => toolHasName(parsedCall.tool_name, tool));
 	if (!tool) {
-		logger.debug(`Model requested tool that does not exist: "${call.tool_name}". Skipping tool...`);
-		return;
+		logger.debug(
+			`Model requested tool that does not exist: "${parsedCall.tool_name}". Skipping tool...`
+		);
+		return undefined;
 	}
 
 	const parametersWithDefaults: Record<string, string> = {};
 
 	for (const input of tool.inputs) {
-		const value = call.parameters[input.name];
+		const value = parsedCall.parameters[input.name];
 
 		// Required so ensure it's there, otherwise return undefined
 		if (input.paramType === "required") {
 			if (value === undefined) {
 				logger.debug(
-					`Model requested tool "${call.tool_name}" but was missing required parameter "${input.name}". Skipping tool...`
+					`Model requested tool "${parsedCall.tool_name}" but was missing required parameter "${input.name}". Skipping tool...`
 				);
 				return;
 			}
@@ -285,7 +303,41 @@ function externalToToolCall(call: ExternalToolCall, tools: Tool[]): ToolCall | u
 	}
 
 	return {
-		name: call.tool_name,
+		name: parsedCall.tool_name,
 		parameters: parametersWithDefaults,
 	};
 }
+
+// Helper functions
+function isValidCallObject(call: unknown): call is Record<string, unknown> {
+	return typeof call === "object" && call !== null;
+}
+
+function parseExternalCall(callObj: Record<string, unknown>) {
+	const nameFields = ["tool_name", "name"] as const;
+	const parametersFields = ["parameters", "arguments", "parameter_definitions"] as const;
+
+	const groupedCall = {
+		tool_name: "" as string,
+		parameters: undefined as Record<string, string> | undefined,
+	};
+
+	for (const name of nameFields) {
+		if (callObj[name]) {
+			groupedCall.tool_name = callObj[name] as string;
+		}
+	}
+
+	for (const name of parametersFields) {
+		if (callObj[name]) {
+			groupedCall.parameters = callObj[name] as Record<string, string>;
+		}
+	}
+
+	return z
+		.object({
+			tool_name: z.string(),
+			parameters: z.record(z.any()),
+		})
+		.parse(groupedCall);
+}

src/lib/server/websearch/search/endpoints/bing.ts

@@ -27,8 +27,6 @@ export default async function search(query: string): Promise<WebSearchSource[]>
 		);
 	}
 
-	console.log(data["webPages"]?.["value"]);
-
 	// Adapt the data structure from the Bing response to match the WebSearchSource type
 	const webPages = data["webPages"]?.["value"] ?? [];
 	return webPages.map((page: any) => ({