Repository: huggingface/transformers
Pull request: fix(benchmarks): prevent version conflicts and ensure proper cleanup in continuous batching

The CI was failing with `TypeError: ContinuousBatchingManager.add_requests() got an unexpected keyword argument 'streaming'` due to a stale PyPI installation being used instead of the local source. Additionally, exceptions during benchmark execution left the manager running, causing downstream OOM errors.

## Changes

- **Remove transformers from benchmark requirements**: The workflow explicitly installs from source in editable mode, but `benchmark_v2/requirements.txt` was installing from PyPI first, creating version conflicts
- **Add try-finally cleanup**: Wrap `time_generate_batch()` logic to ensure `manager.stop()` is always called, preventing GPU memory leaks when exceptions occur

```python
manager = self.model.init_continuous_batching(config)
manager.start()
try:
    # benchmark logic
    manager.add_requests(inputs, max_new_tokens=max_new_tokens, streaming=True)
    # ...
finally:
    manager.stop()  # Always cleanup, even on exception
```

<!-- START COPILOT CODING AGENT TIPS -->
---

✨ Let Copilot coding agent [set things up for you](https://github.com/huggingface/transformers/issues/new?title=✨+Set+up+Copilot+instructions&body=Configure%20instructions%20for%20this%20repository%20as%20documented%20in%20%5BBest%20practices%20for%20Copilot%20coding%20agent%20in%20your%20repository%5D%28https://gh.io/copilot-coding-agent-tips%29%2E%0A%0A%3COnboard%20this%20repo%3E&assignees=copilot) — coding agent works faster and does higher quality work when set up for your repo.


Changed files:
filename: benchmark_v2/framework/benchmark_runner.py
patch: @@ -291,37 +291,39 @@ def time_generate_batch(
         )
         manager = self.model.init_continuous_batching(config)
         manager.start()
-        first_req_results = []
-        timestamps = []
-        wall_time_0 = time.perf_counter()
-        inputs = self.inputs["input_ids"].tolist()
-        manager.add_requests(inputs, max_new_tokens=max_new_tokens, streaming=True)
-        first_req_id = None
-        num_requests = len(inputs)
-        finished_requests = 0
-        while finished_requests < num_requests:
-            # NOTE: I don't like having the extra if stmt here, but hopefully won't degrade perf too much
-            result = manager.get_result()
-            if result:
-                timestamps.append(time.perf_counter() - wall_time_0)
-                if result.is_finished():
-                    finished_requests += 1
-                if first_req_id is None:
-                    first_req_id = result.request_id
-                if result.request_id == first_req_id:
-                    first_req_results.append(result)
-            else:
-                if not manager.is_running():
-                    raise RuntimeError("Generation thread exited unexpectedly")
-        wall_time_1 = time.perf_counter()
-        manager.stop()
-        gpu_metrics = gpu_monitor.stop_and_collect() if gpu_monitor is not None else None
-        decoded_output = self.tokenizer.decode(
-            [res.generated_tokens[0] for res in first_req_results], skip_special_tokens=True
-        )
-        shape_and_decoded_output = f"{(1, len(first_req_results))} | {decoded_output}"
-        e2e_latency = wall_time_1 - wall_time_0
-        return e2e_latency, timestamps, shape_and_decoded_output, gpu_metrics
+        try:
+            first_req_results = []
+            timestamps = []
+            wall_time_0 = time.perf_counter()
+            inputs = self.inputs["input_ids"].tolist()
+            manager.add_requests(inputs, max_new_tokens=max_new_tokens, streaming=True)
+            first_req_id = None
+            num_requests = len(inputs)
+            finished_requests = 0
+            while finished_requests < num_requests:
+                # NOTE: I don't like having the extra if stmt here, but hopefully won't degrade perf too much
+                result = manager.get_result()
+                if result:
+                    timestamps.append(time.perf_counter() - wall_time_0)
+                    if result.is_finished():
+                        finished_requests += 1
+                    if first_req_id is None:
+                        first_req_id = result.request_id
+                    if result.request_id == first_req_id:
+                        first_req_results.append(result)
+                else:
+                    if not manager.is_running():
+                        raise RuntimeError("Generation thread exited unexpectedly")
+            wall_time_1 = time.perf_counter()
+            gpu_metrics = gpu_monitor.stop_and_collect() if gpu_monitor is not None else None
+            decoded_output = self.tokenizer.decode(
+                [res.generated_tokens[0] for res in first_req_results], skip_special_tokens=True
+            )
+            shape_and_decoded_output = f"{(1, len(first_req_results))} | {decoded_output}"
+            e2e_latency = wall_time_1 - wall_time_0
+            return e2e_latency, timestamps, shape_and_decoded_output, gpu_metrics
+        finally:
+            manager.stop()
 
     def time_generate(
         self,

filename: benchmark_v2/requirements.txt
patch: @@ -2,6 +2,5 @@ numpy>=1.21.0
 psutil>=5.8.0
 gpustat>=1.0.0
 torch>=2.0.0
-transformers>=4.30.0
 datasets>=2.10.0
 huggingface_hub>=0.16.0