Google Vertex API support (#950)

* Added Google Vertex API support

Google Application credentials can be set via

GOOGLE_APPLICATION_CREDENTIALS = clientid.json

* Updated the readme file

* Updated the package-lock file

* Updated the package-lock file
Fixed minor issues

* Moved "@google-cloud/vertexai" to optional dependencies

* Updated for lint checker

README.md

@@ -528,6 +528,42 @@ You can also set `"service" : "lambda"` to use a lambda instance.
 
 You can get the `accessKey` and `secretKey` from your AWS user, under programmatic access.
 
+##### Google Vertex models
+
+Chat UI can connect to the google Vertex API endpoints ([List of supported models](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models)).
+
+To enable:
+
+1.  [Select](https://console.cloud.google.com/project) or [create](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project) a Google Cloud project.
+1.  [Enable billing for your project](https://cloud.google.com/billing/docs/how-to/modify-project).
+1.  [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).
+1.  [Set up authentication with a service account](https://cloud.google.com/docs/authentication/getting-started)
+    so you can access the API from your local workstation.
+
+The service account credentials file can be imported as an environmental variable:
+
+```env
+    GOOGLE_APPLICATION_CREDENTIALS = clientid.json
+```
+
+Make sure docker has access to the file. Afterwards Google Vertex endpoints can be configured as following:
+
+```
+MODELS=`[
+//...
+    {
+       "name": "gemini-1.0-pro", //model-name
+       "displayName": "Vertex Gemini Pro 1.0",
+       "location": "europe-west3",
+       "apiEndpoint": "", //alternative api endpoint url
+       "endpoints" : [{
+         "type": "vertex"
+       }]
+     },
+]`
+
+```
+
 ### Custom endpoint authorization
 
 #### Basic and Bearer

package-lock.json

@@ -74,7 +74,8 @@
 			"optionalDependencies": {
 				"@anthropic-ai/sdk": "^0.17.1",
 				"aws4fetch": "^1.0.17",
-				"openai": "^4.14.2"
+				"openai": "^4.14.2",
+				"@google-cloud/vertexai": "^0.5.0"
 			}
 		},
 		"node_modules/@alloc/quick-lru": {
@@ -628,6 +629,17 @@
 				"node": ">=14"
 			}
 		},
+		"node_modules/@google-cloud/vertexai": {
+			"version": "0.5.0",
+			"resolved": "https://registry.npmjs.org/@google-cloud/vertexai/-/vertexai-0.5.0.tgz",
+			"integrity": "sha512-qIFHYTXA5UCLdm9JG+Xf1suomCXxRqa1PKdYjqXuhZsCm8mn37Rb0Tf8djlhDzuRVWyWoQTmsWpsk28ZTmbqJg==",
+			"dependencies": {
+				"google-auth-library": "^9.1.0"
+			},
+			"engines": {
+				"node": ">=18.0.0"
+			}
+		},
 		"node_modules/@huggingface/hub": {
 			"version": "0.5.1",
 			"resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-0.5.1.tgz",
@@ -2679,6 +2691,14 @@
 				}
 			]
 		},
+		"node_modules/bignumber.js": {
+			"version": "9.1.2",
+			"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz",
+			"integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==",
+			"engines": {
+				"node": "*"
+			}
+		},
 		"node_modules/binary-extensions": {
 			"version": "2.2.0",
 			"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -2809,6 +2829,11 @@
 				"node": "*"
 			}
 		},
+		"node_modules/buffer-equal-constant-time": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
+			"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
+		},
 		"node_modules/bufferutil": {
 			"version": "4.0.7",
 			"resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz",
@@ -3433,6 +3458,14 @@
 				"node": ">=12"
 			}
 		},
+		"node_modules/ecdsa-sig-formatter": {
+			"version": "1.0.11",
+			"resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
+			"integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
+			"dependencies": {
+				"safe-buffer": "^5.0.1"
+			}
+		},
 		"node_modules/electron-to-chromium": {
 			"version": "1.4.359",
 			"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.359.tgz",
@@ -3787,6 +3820,11 @@
 				"node": ">=6"
 			}
 		},
+		"node_modules/extend": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+			"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
+		},
 		"node_modules/fast-deep-equal": {
 			"version": "3.1.3",
 			"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -3993,6 +4031,55 @@
 				"url": "https://github.com/sponsors/ljharb"
 			}
 		},
+		"node_modules/gaxios": {
+			"version": "6.3.0",
+			"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.3.0.tgz",
+			"integrity": "sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==",
+			"dependencies": {
+				"extend": "^3.0.2",
+				"https-proxy-agent": "^7.0.1",
+				"is-stream": "^2.0.0",
+				"node-fetch": "^2.6.9"
+			},
+			"engines": {
+				"node": ">=14"
+			}
+		},
+		"node_modules/gaxios/node_modules/agent-base": {
+			"version": "7.1.0",
+			"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz",
+			"integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==",
+			"dependencies": {
+				"debug": "^4.3.4"
+			},
+			"engines": {
+				"node": ">= 14"
+			}
+		},
+		"node_modules/gaxios/node_modules/https-proxy-agent": {
+			"version": "7.0.4",
+			"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz",
+			"integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==",
+			"dependencies": {
+				"agent-base": "^7.0.2",
+				"debug": "4"
+			},
+			"engines": {
+				"node": ">= 14"
+			}
+		},
+		"node_modules/gcp-metadata": {
+			"version": "6.1.0",
+			"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz",
+			"integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==",
+			"dependencies": {
+				"gaxios": "^6.0.0",
+				"json-bigint": "^1.0.0"
+			},
+			"engines": {
+				"node": ">=14"
+			}
+		},
 		"node_modules/get-func-name": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
@@ -4097,6 +4184,22 @@
 			"integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==",
 			"dev": true
 		},
+		"node_modules/google-auth-library": {
+			"version": "9.7.0",
+			"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.7.0.tgz",
+			"integrity": "sha512-I/AvzBiUXDzLOy4iIZ2W+Zq33W4lcukQv1nl7C8WUA6SQwyQwUwu3waNmWNAvzds//FG8SZ+DnKnW/2k6mQS8A==",
+			"dependencies": {
+				"base64-js": "^1.3.0",
+				"ecdsa-sig-formatter": "^1.0.11",
+				"gaxios": "^6.1.1",
+				"gcp-metadata": "^6.1.0",
+				"gtoken": "^7.0.0",
+				"jws": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=14"
+			}
+		},
 		"node_modules/graceful-fs": {
 			"version": "4.2.11",
 			"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -4115,6 +4218,18 @@
 			"integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
 			"dev": true
 		},
+		"node_modules/gtoken": {
+			"version": "7.1.0",
+			"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz",
+			"integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==",
+			"dependencies": {
+				"gaxios": "^6.0.0",
+				"jws": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=14.0.0"
+			}
+		},
 		"node_modules/guid-typescript": {
 			"version": "1.0.9",
 			"resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
@@ -4460,7 +4575,6 @@
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
 			"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
-			"dev": true,
 			"engines": {
 				"node": ">=8"
 			},
@@ -4590,6 +4704,14 @@
 				"node": ">=14"
 			}
 		},
+		"node_modules/json-bigint": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
+			"integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==",
+			"dependencies": {
+				"bignumber.js": "^9.0.0"
+			}
+		},
 		"node_modules/json-schema-traverse": {
 			"version": "0.4.1",
 			"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -4619,6 +4741,25 @@
 			"integrity": "sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==",
 			"dev": true
 		},
+		"node_modules/jwa": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
+			"integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
+			"dependencies": {
+				"buffer-equal-constant-time": "1.0.1",
+				"ecdsa-sig-formatter": "1.0.11",
+				"safe-buffer": "^5.0.1"
+			}
+		},
+		"node_modules/jws": {
+			"version": "4.0.0",
+			"resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
+			"integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
+			"dependencies": {
+				"jwa": "^2.0.0",
+				"safe-buffer": "^5.0.1"
+			}
+		},
 		"node_modules/katex": {
 			"version": "0.16.10",
 			"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz",
@@ -5152,7 +5293,6 @@
 			"version": "2.7.0",
 			"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
 			"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
-			"optional": true,
 			"dependencies": {
 				"whatwg-url": "^5.0.0"
 			},
@@ -5171,20 +5311,17 @@
 		"node_modules/node-fetch/node_modules/tr46": {
 			"version": "0.0.3",
 			"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-			"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
-			"optional": true
+			"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
 		},
 		"node_modules/node-fetch/node_modules/webidl-conversions": {
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-			"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
-			"optional": true
+			"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
 		},
 		"node_modules/node-fetch/node_modules/whatwg-url": {
 			"version": "5.0.0",
 			"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
 			"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-			"optional": true,
 			"dependencies": {
 				"tr46": "~0.0.3",
 				"webidl-conversions": "^3.0.0"

package.json

@@ -84,6 +84,7 @@
 	"optionalDependencies": {
 		"@anthropic-ai/sdk": "^0.17.1",
 		"aws4fetch": "^1.0.17",
-		"openai": "^4.14.2"
+		"openai": "^4.14.2",
+		"@google-cloud/vertexai": "^0.5.0"
 	}
 }

src/lib/server/endpoints/endpoints.ts

@@ -6,6 +6,8 @@ import endpointAws, { endpointAwsParametersSchema } from "./aws/endpointAws";
 import { endpointOAIParametersSchema, endpointOai } from "./openai/endpointOai";
 import endpointLlamacpp, { endpointLlamacppParametersSchema } from "./llamacpp/endpointLlamacpp";
 import endpointOllama, { endpointOllamaParametersSchema } from "./ollama/endpointOllama";
+import endpointVertex, { endpointVertexParametersSchema } from "./google/endpointVertex";
+
 import {
 	endpointAnthropic,
 	endpointAnthropicParametersSchema,
@@ -39,6 +41,7 @@ export const endpoints = {
 	openai: endpointOai,
 	llamacpp: endpointLlamacpp,
 	ollama: endpointOllama,
+	vertex: endpointVertex,
 };
 
 export const endpointSchema = z.discriminatedUnion("type", [
@@ -48,5 +51,6 @@ export const endpointSchema = z.discriminatedUnion("type", [
 	endpointTgiParametersSchema,
 	endpointLlamacppParametersSchema,
 	endpointOllamaParametersSchema,
+	endpointVertexParametersSchema,
 ]);
 export default endpoints;

src/lib/server/endpoints/google/endpointVertex.ts

@@ -0,0 +1,78 @@
+import { VertexAI, HarmCategory, HarmBlockThreshold } from "@google-cloud/vertexai";
+import { buildPrompt } from "$lib/buildPrompt";
+import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import type { Endpoint } from "../endpoints";
+import { z } from "zod";
+
+export const endpointVertexParametersSchema = z.object({
+	weight: z.number().int().positive().default(1),
+	model: z.any(), // allow optional and validate against emptiness
+	type: z.literal("vertex"),
+	location: z.string().default("europe-west1"),
+	project: z.string(),
+	apiEndpoint: z.string().optional(),
+});
+
+export function endpointVertex(input: z.input<typeof endpointVertexParametersSchema>): Endpoint {
+	const { project, location, model, apiEndpoint } = endpointVertexParametersSchema.parse(input);
+
+	const vertex_ai = new VertexAI({
+		project,
+		location,
+		apiEndpoint,
+	});
+
+	const generativeModel = vertex_ai.getGenerativeModel({
+		model: model.id ?? model.name,
+		safety_settings: [
+			{
+				category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+				threshold: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+			},
+		],
+		generation_config: {},
+	});
+
+	return async ({ messages, preprompt, continueMessage }) => {
+		const prompt = await buildPrompt({
+			messages,
+			continueMessage,
+			preprompt,
+			model,
+		});
+
+		const chat = generativeModel.startChat();
+		const result = await chat.sendMessageStream(prompt);
+		let tokenId = 0;
+
+		return (async function* () {
+			let generatedText = "";
+
+			for await (const data of result.stream) {
+				if (Array.isArray(data?.candidates) && data.candidates.length > 0) {
+					const firstPart = data.candidates[0].content.parts[0];
+					if ("text" in firstPart) {
+						const content = firstPart.text;
+						generatedText += content;
+						const output: TextGenerationStreamOutput = {
+							token: {
+								id: tokenId++,
+								text: content ?? "",
+								logprob: 0,
+								special: false,
+							},
+							generated_text: generatedText,
+							details: null,
+						};
+						yield output;
+					}
+
+					if (!data.candidates.slice(-1)[0].finishReason) break;
+				} else {
+					break;
+				}
+			}
+		})();
+	};
+}
+export default endpointVertex;

src/lib/server/models.ts

@@ -128,6 +128,8 @@ const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
 						return endpoints.llamacpp(args);
 					case "ollama":
 						return endpoints.ollama(args);
+					case "vertex":
+						return await endpoints.vertex(args);
 					default:
 						// for legacy reason
 						return endpoints.tgi(args);