Repository: huggingface/huggingface_hub
Pull request #2136 (2024-03-20 10:37:57)

Fix InferenceClient.text_generation for non-tgi models
Fix https://github.com/huggingface/huggingface_hub/issues/2135 cc @anakin87

HuggingFaceDocBuilderDev: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/huggingface_hub/pr_2136). All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.
Wauplin: Thanks for the quick review!

Patch:
diff --git a/src/huggingface_hub/inference/_client.py b/src/huggingface_hub/inference/_client.py
index 6a07ef65..aed4643b 100644
--- a/src/huggingface_hub/inference/_client.py
+++ b/src/huggingface_hub/inference/_client.py
@@ -1787,7 +1787,7 @@ class InferenceClient:
             parameters: Dict = payload["parameters"]  # type: ignore [assignment]
 
             ignored_parameters = []
-            for key in "watermark", "details", "decoder_input_details", "best_of", "stop":
+            for key in "watermark", "details", "decoder_input_details", "best_of", "stop", "return_full_text":
                 if parameters[key] is not None:
                     ignored_parameters.append(key)
                 del parameters[key]
diff --git a/src/huggingface_hub/inference/_generated/_async_client.py b/src/huggingface_hub/inference/_generated/_async_client.py
index 1b695b71..ac22e3af 100644
--- a/src/huggingface_hub/inference/_generated/_async_client.py
+++ b/src/huggingface_hub/inference/_generated/_async_client.py
@@ -1812,7 +1812,7 @@ class AsyncInferenceClient:
             parameters: Dict = payload["parameters"]  # type: ignore [assignment]
 
             ignored_parameters = []
-            for key in "watermark", "details", "decoder_input_details", "best_of", "stop":
+            for key in "watermark", "details", "decoder_input_details", "best_of", "stop", "return_full_text":
                 if parameters[key] is not None:
                     ignored_parameters.append(key)
                 del parameters[key]


Test patch:
diff --git a/tests/cassettes/TestTextGenerationClientVCR.test_generate_non_tgi_endpoint_regression_test.yaml b/tests/cassettes/TestTextGenerationClientVCR.test_generate_non_tgi_endpoint_regression_test.yaml
new file mode 100644
index 00000000..cda4c9a3
--- /dev/null
+++ b/tests/cassettes/TestTextGenerationClientVCR.test_generate_non_tgi_endpoint_regression_test.yaml
@@ -0,0 +1,112 @@
+interactions:
+- request:
+    body: '{"inputs": "How are you today?", "parameters": {"best_of": null, "decoder_input_details":
+      false, "details": false, "do_sample": false, "max_new_tokens": 20, "repetition_penalty":
+      null, "return_full_text": false, "seed": null, "stop": [], "temperature": null,
+      "top_k": null, "top_p": null, "truncate": null, "typical_p": null, "watermark":
+      false}, "stream": false}'
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '363'
+      Content-Type:
+      - application/json
+      X-Amzn-Trace-Id:
+      - fb7085c4-07d9-47c5-bbf9-bdc73720398f
+      user-agent:
+      - unknown/None; hf_hub/0.22.0.dev0; python/3.10.12; torch/2.2.0; tensorflow/2.15.0.post1;
+        fastcore/1.5.23
+    method: POST
+    uri: https://api-inference.huggingface.co/models/google/flan-t5-large
+  response:
+    body:
+      string: '{"error":"The following `model_kwargs` are not used by the model: [''return_full_text'',
+        ''watermark'', ''stop'', ''decoder_input_details'', ''details''] (note: typos
+        in the generate arguments will also show up in this list)","warnings":["There
+        was an inference error: The following `model_kwargs` are not used by the model:
+        [''return_full_text'', ''watermark'', ''stop'', ''decoder_input_details'',
+        ''details''] (note: typos in the generate arguments will also show up in this
+        list)"]}'
+    headers:
+      Connection:
+      - keep-alive
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 20 Mar 2024 10:34:43 GMT
+      Transfer-Encoding:
+      - chunked
+      access-control-allow-credentials:
+      - 'true'
+      access-control-expose-headers:
+      - x-compute-type, x-compute-time
+      server:
+      - uvicorn
+      vary:
+      - Origin, Access-Control-Request-Method, Access-Control-Request-Headers
+      x-compute-time:
+      - '0.002'
+      x-compute-type:
+      - cpu
+      x-request-id:
+      - UzkyAdcrW8iGZ1G22NaQm
+      x-sha:
+      - 0613663d0d48ea86ba8cb3d7a44f0f65dc596a2a
+    status:
+      code: 400
+      message: Bad Request
+- request:
+    body: '{"inputs": "How are you today?", "parameters": {"do_sample": false, "max_new_tokens":
+      20, "repetition_penalty": null, "seed": null, "temperature": null, "top_k":
+      null, "top_p": null, "truncate": null, "typical_p": null}, "stream": false}'
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '237'
+      Content-Type:
+      - application/json
+      X-Amzn-Trace-Id:
+      - e7ffd399-0e70-4178-9939-12ceb0f6fedb
+      user-agent:
+      - unknown/None; hf_hub/0.22.0.dev0; python/3.10.12; torch/2.2.0; tensorflow/2.15.0.post1;
+        fastcore/1.5.23
+    method: POST
+    uri: https://api-inference.huggingface.co/models/google/flan-t5-large
+  response:
+    body:
+      string: '[{"generated_text":"I am at work"}]'
+    headers:
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '35'
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 20 Mar 2024 10:34:43 GMT
+      access-control-allow-credentials:
+      - 'true'
+      vary:
+      - Origin, Access-Control-Request-Method, Access-Control-Request-Headers
+      x-compute-time:
+      - '0.434'
+      x-compute-type:
+      - cache
+      x-request-id:
+      - 0caafN63HGaXbHkujJms2
+      x-sha:
+      - 0613663d0d48ea86ba8cb3d7a44f0f65dc596a2a
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/test_inference_text_generation.py b/tests/test_inference_text_generation.py
index 76bf35ee..c2fcd678 100644
--- a/tests/test_inference_text_generation.py
+++ b/tests/test_inference_text_generation.py
@@ -135,3 +135,11 @@ class TestTextGenerationClientVCR(unittest.TestCase):
         # Return as stream raises error
         with self.assertRaises(ValueError):
             self.client.text_generation("0 1 2", model="gpt2", max_new_tokens=10, stream=True)
+
+    def test_generate_non_tgi_endpoint_regression_test(self):
+        # Regression test for https://github.com/huggingface/huggingface_hub/issues/2135
+        with self.assertWarnsRegex(UserWarning, "Ignoring parameters .* 'return_full_text'"):
+            text = self.client.text_generation(
+                prompt="How are you today?", max_new_tokens=20, model="google/flan-t5-large"
+            )
+        assert text == "I am at work"