Repository: nginx/nginx-gateway-fabric

Issue:
Inference Extension: Extract model name from request body
In order to support model redirects and traffic splitting, NGINX needs to be able to extract the model name from the client request body. It will then get used to set the `X-Gateway-Model-Name` header in the request to the Endpoint Picker (EPP) for it to make a decision on which endpoint to route the client request to.

This story is to simply write the module to extract the model name. A followup story will actually use this.

Acceptance Criteria:
- Create an NJS module that can extract the model name from the client request body
- The NJS module should be built in to the NGINX container images

Developer Notes:
- OpenAI uses a request body that should look like the following when a client makes a request to an AI workload:

```json
{
  "model": "gpt-4o",
  "messages": [
    {
      "role": "user",
      "content": "Some question..."
    }
  ]
}
```

Design doc: https://github.com/nginx/nginx-gateway-fabric/blob/main/docs/proposals/gateway-inference-extension.md

Discussion:

Solution patch:
diff --git a/.nvmrc b/.nvmrc
index 2bd5a0a9..a45fd52c 100644
--- a/.nvmrc
+++ b/.nvmrc
@@ -1 +1 @@
-22
+24
diff --git a/Makefile b/Makefile
index cc408afc..a49147a7 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ GEN_CRD_API_REFERENCE_DOCS_VERSION = v0.3.0
 # renovate: datasource=go depName=sigs.k8s.io/controller-tools
 CONTROLLER_TOOLS_VERSION = v0.19.0
 # renovate: datasource=docker depName=node
-NODE_VERSION = 22
+NODE_VERSION = 24
 # renovate: datasource=docker depName=quay.io/helmpack/chart-testing
 CHART_TESTING_VERSION = v3.13.0
 # renovate: datasource=github-tags depName=dadav/helm-schema
diff --git a/build/Dockerfile.nginx b/build/Dockerfile.nginx
index 83c7e066..d326c089 100644
--- a/build/Dockerfile.nginx
+++ b/build/Dockerfile.nginx
@@ -23,7 +23,7 @@ RUN apk add --no-cache bash \
     && ln -sf /dev/stderr /var/log/nginx/error.log
 
 COPY build/entrypoint.sh /agent/entrypoint.sh
-COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NJS_DIR}/ /usr/lib/nginx/modules/njs/
 COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
diff --git a/build/Dockerfile.nginxplus b/build/Dockerfile.nginxplus
index a33adefb..3828ce39 100644
--- a/build/Dockerfile.nginxplus
+++ b/build/Dockerfile.nginxplus
@@ -29,7 +29,7 @@ RUN apk add --no-cache bash \
     && ln -sf /dev/stderr /var/log/nginx/error.log
 
 COPY build/entrypoint.sh /agent/entrypoint.sh
-COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NJS_DIR}/ /usr/lib/nginx/modules/njs/
 COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
diff --git a/internal/controller/nginx/conf/nginx-plus.conf b/internal/controller/nginx/conf/nginx-plus.conf
index f2b0ec0d..50ba9f97 100644
--- a/internal/controller/nginx/conf/nginx-plus.conf
+++ b/internal/controller/nginx/conf/nginx-plus.conf
@@ -13,6 +13,7 @@ http {
   include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/mime.types;
   js_import /usr/lib/nginx/modules/njs/httpmatches.js;
+  js_import /usr/lib/nginx/modules/njs/epp.js;
 
   default_type application/octet-stream;
 
diff --git a/internal/controller/nginx/conf/nginx.conf b/internal/controller/nginx/conf/nginx.conf
index 791994fd..6c4f6be8 100644
--- a/internal/controller/nginx/conf/nginx.conf
+++ b/internal/controller/nginx/conf/nginx.conf
@@ -13,6 +13,7 @@ http {
   include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/mime.types;
   js_import /usr/lib/nginx/modules/njs/httpmatches.js;
+  js_import /usr/lib/nginx/modules/njs/epp.js;
 
   default_type application/octet-stream;
 
diff --git a/internal/controller/nginx/modules/README.md b/internal/controller/nginx/modules/README.md
index 9c7c8052..3313ea66 100644
--- a/internal/controller/nginx/modules/README.md
+++ b/internal/controller/nginx/modules/README.md
@@ -22,6 +22,7 @@ dependencies.
 
 - [httpmatches](./src/httpmatches.js): a location handler for HTTP requests. It redirects requests to an internal
   location block based on the request's headers, arguments, and method.
+- [epp](./src/epp.js): handles communication with the EndpointPicker (EPP) component. This is for acquiring a specific AI endpoint to route client traffic to when using the Gateway API Inference Extension.
 
 ### Helpful Resources for Module Development
 
diff --git a/internal/controller/nginx/modules/src/epp.js b/internal/controller/nginx/modules/src/epp.js
new file mode 100644
index 00000000..8efcd70e
--- /dev/null
+++ b/internal/controller/nginx/modules/src/epp.js
@@ -0,0 +1,29 @@
+// This file contains the methods to get an AI workload endpoint from the EndpointPicker (EPP).
+
+// TODO (sberman): this module will need to be enhanced to include the following:
+// - function that sends the subrequest to the Go middleware application (to get the endpoint from EPP)
+// - if a user has specified an Exact matching condition for a model name, extract the model name from
+// the request body, and if it matches that condition, set the proper value in the X-Gateway-Model-Name header
+// (based on if we do a redirect or traffic split (see design doc)) in the subrequest. If the client request
+// already has this header set, then I don't think we need to extract the model from the body, just pass
+// through the existing header.
+// I believe we have to use js_content to call the NJS functionality. Because this takes over
+// the request, we will likely have to finish the NJS functionality with an internalRedirect to an internal
+// location that proxy_passes to the chosen endpoint.
+
+// extractModel extracts the model name from the request body.
+function extractModel(r) {
+	try {
+		var body = JSON.parse(r.requestText);
+		if (body && body.model !== undefined) {
+			return String(body.model);
+		}
+	} catch (e) {
+		r.error(`error parsing request body for model name: ${e.message}`);
+		return '';
+	}
+	r.error('request body does not contain model parameter');
+	return '';
+}
+
+export default { extractModel };



Test patch:
diff --git a/internal/controller/nginx/modules/test/epp.test.js b/internal/controller/nginx/modules/test/epp.test.js
new file mode 100644
index 00000000..6994423e
--- /dev/null
+++ b/internal/controller/nginx/modules/test/epp.test.js
@@ -0,0 +1,52 @@
+import { default as epp } from '../src/epp.js';
+import { expect, describe, it } from 'vitest';
+
+function makeRequest(body) {
+	let r = {
+		// Test mocks
+		error(msg) {
+			r.variables.error = msg;
+		},
+		requestText: body,
+		variables: {},
+	};
+
+	return r;
+}
+
+describe('extractModel', () => {
+	const tests = [
+		{
+			name: 'returns the model value',
+			body: '{"model":"gpt-4"}',
+			model: 'gpt-4',
+			error: undefined,
+		},
+		{
+			name: 'returns empty string if model is missing',
+			body: '{"foo":1}',
+			model: '',
+			error: 'request body does not contain model parameter',
+		},
+		{
+			name: 'returns empty string for invalid JSON',
+			body: 'not-json',
+			model: '',
+			error: `error parsing request body for model name: Unexpected token 'o', "not-json" is not valid JSON`,
+		},
+		{
+			name: 'empty request body',
+			body: '',
+			model: '',
+			error: 'error parsing request body for model name: Unexpected end of JSON input',
+		},
+	];
+
+	tests.forEach((test) => {
+		it(test.name, () => {
+			let r = makeRequest(test.body);
+			expect(epp.extractModel(r)).to.equal(test.model);
+			expect(r.variables.error).to.equal(test.error);
+		});
+	});
+});