Repository: Graylog2/graylog2-server (language: java)

Issue:
"Caught an unhandled exception while executing event processor" when search request to OpenSearch has to many indices
## Expected Behavior
The event creation should either work as expected or tell the user why it failed so he can take countermeasures.

## Current Behavior
The event creation fails with an error which is not visible to the user in the frontend. An error is logged on the graylog application log:
```
2025-02-06 10:11:40,498 ERROR: org.graylog.events.processor.EventProcessorEngine - Caught an unhandled exception while executing event processor <aggregation event name> - Make sure to modify the event processor to throw only EventProcessorException so we get more context!
org.graylog.shaded.opensearch2.org.opensearch.OpenSearchException: Unable to retrieve next chunk from search: 
	at org.graylog.storage.opensearch2.OpenSearchClient.exceptionFrom(OpenSearchClient.java:212) ~[?:?]
	at org.graylog.storage.opensearch2.OpenSearchClient.executeWithIOException(OpenSearchClient.java:167) ~[?:?]
	at org.graylog.storage.opensearch2.PaginationResultOS2.nextSearchResult(PaginationResultOS2.java:56) ~[?:?]
	at org.graylog.storage.opensearch2.PaginationResultOS2.nextSearchResult(PaginationResultOS2.java:31) ~[?:?]
	at org.graylog2.indexer.results.ChunkedQueryResult.nextChunk(ChunkedQueryResult.java:68) ~[graylog.jar:?]
	at org.graylog.storage.opensearch2.MoreSearchAdapterOS2.scrollEvents(MoreSearchAdapterOS2.java:170) ~[?:?]
	at org.graylog.events.search.MoreSearch.scrollQuery(MoreSearch.java:157) ~[graylog.jar:?]
	at org.graylog.events.processor.aggregation.AggregationEventProcessor.filterSearch(AggregationEventProcessor.java:285) ~[graylog.jar:?]
	at org.graylog.events.processor.aggregation.AggregationEventProcessor.createEvents(AggregationEventProcessor.java:146) ~[graylog.jar:?]
	at org.graylog.events.processor.EventProcessorEngine.execute(EventProcessorEngine.java:100) ~[graylog.jar:?]
	at org.graylog.events.processor.EventProcessorExecutionJob.execute(EventProcessorExecutionJob.java:115) ~[graylog.jar:?]
	at org.graylog.scheduler.JobExecutionEngine.executeJob(JobExecutionEngine.java:292) ~[graylog.jar:?]
	at org.graylog.scheduler.JobExecutionEngine.lambda$handleTrigger$4(JobExecutionEngine.java:265) ~[graylog.jar:?]
	at com.codahale.metrics.Timer.time(Timer.java:151) ~[graylog.jar:?]
	at org.graylog.scheduler.JobExecutionEngine.handleTrigger(JobExecutionEngine.java:265) ~[graylog.jar:?]
	at org.graylog.scheduler.JobExecutionEngine.handleTriggerWithConcurrencyLimit(JobExecutionEngine.java:237) ~[graylog.jar:?]
	at org.graylog.scheduler.JobExecutionEngine.lambda$execute$2(JobExecutionEngine.java:202) ~[graylog.jar:?]
	at org.graylog.scheduler.worker.JobWorkerPool.lambda$execute$0(JobWorkerPool.java:115) ~[graylog.jar:?]
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:259) [graylog.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
	at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66) [graylog.jar:?]
	at java.base/java.lang.Thread.run(Thread.java:840) [?:?]
Caused by: org.graylog.shaded.opensearch2.org.opensearch.OpenSearchStatusException: OpenSearch exception [type=too_long_http_line_exception, reason=An HTTP line is larger than 4096 bytes.]
	at org.graylog.shaded.opensearch2.org.opensearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:210) ~[?:?]
	at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:2235) ~[?:?]
	at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:2212) ~[?:?]
	at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1931) ~[?:?]
	at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1884) ~[?:?]
	at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1852) ~[?:?]
	at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1095) ~[?:?]
	at org.graylog.storage.opensearch2.PaginationResultOS2.lambda$nextSearchResult$0(PaginationResultOS2.java:56) ~[?:?]
	at org.graylog.storage.opensearch2.OpenSearchClient.executeWithIOException(OpenSearchClient.java:160) ~[?:?]
	... 21 more
	Suppressed: org.graylog.shaded.opensearch2.org.opensearch.client.ResponseException: method [POST], host [URL TO OPENSEARCH], URI [/graylog_1848,graylog_1847,graylog_warm_1741,graylog_warm_1740,graylog_1849,graylog_warm_1732,graylog_warm_1731,graylog_warm_1734,graylog_warm_1733,graylog_warm_1736,graylog_warm_1735,graylog_warm_1738,graylog_warm_1737,graylog_warm_1739,graylog_warm_1750,graylog_warm_1752,graylog_warm_1751,graylog_warm_1743,graylog_warm_1742,graylog_warm_1745,graylog_warm_1744,graylog_warm_1747,graylog_warm_1746,graylog_warm_1749,graylog_warm_1748,graylog_warm_1840,graylog_warm_1710,graylog_warm_1831,graylog_warm_1830,graylog_warm_1712,graylog_warm_1833,graylog_warm_1711,graylog_warm_1832,graylog_warm_1714,graylog_warm_1835,graylog_warm_1713,graylog_warm_1834,graylog_warm_1716,graylog_warm_1837,graylog_warm_1715,graylog_warm_1836,graylog_warm_1718,graylog_warm_1839,graylog_warm_1717,graylog_warm_1838,graylog_warm_1719,graylog_warm_1730,graylog_warm_1721,graylog_warm_1842,graylog_warm_1720,graylog_warm_1841,graylog_warm_1723,graylog_warm_1844,graylog_warm_1722,graylog_warm_1843,graylog_warm_1725,graylog_warm_1846,graylog_warm_1724,graylog_warm_1845,graylog_warm_1727,graylog_warm_1726,graylog_warm_1729,graylog_warm_1728,graylog_warm_1811,graylog_warm_1810,graylog_warm_1813,graylog_warm_1812,graylog_warm_1815,graylog_warm_1814,graylog_warm_1817,graylog_warm_1816,graylog_warm_1819,graylog_warm_1818,graylog_warm_1820,graylog_warm_1701,graylog_warm_1822,graylog_warm_1700,graylog_warm_1821,graylog_warm_1703,graylog_warm_1824,graylog_warm_1702,graylog_warm_1823,graylog_warm_1705,graylog_warm_1826,graylog_warm_1704,graylog_warm_1825,graylog_warm_1707,graylog_warm_1828,graylog_warm_1706,graylog_warm_1827,graylog_warm_1709,graylog_warm_1708,graylog_warm_1829,graylog_1862,graylog_1861,graylog_1863,graylog_1855,graylog_1854,graylog_1857,graylog_1856,graylog_1851,graylog_1850,graylog_1853,graylog_1852,graylog_1859,graylog_1858,graylog_warm_1800,graylog_warm_1802,graylog_warm_1801,graylog_warm_1804,graylog_warm_1803,graylog_warm_1806,graylog_warm_1805,graylog_warm_1808,graylog_1860,graylog_warm_1807,graylog_warm_1809,graylog_warm_1699,graylog_warm_1698,graylog_warm_1680,graylog_warm_1682,graylog_warm_1681,graylog_warm_1684,graylog_warm_1683,graylog_warm_1686,graylog_warm_1685,graylog_warm_1677,graylog_warm_1798,graylog_warm_1676,graylog_warm_1797,graylog_warm_1679,graylog_warm_1678,graylog_warm_1799,graylog_warm_1691,graylog_warm_1690,graylog_warm_1693,graylog_warm_1692,graylog_warm_1695,graylog_warm_1694,graylog_warm_1697,graylog_warm_1696,graylog_warm_1688,graylog_warm_1687,graylog_warm_1689,graylog_warm_1660,graylog_warm_1781,graylog_warm_1780,graylog_warm_1662,graylog_warm_1783,graylog_warm_1661,graylog_warm_1782,graylog_warm_1664,graylog_warm_1785,graylog_warm_1663,graylog_warm_1784,graylog_warm_1655,graylog_warm_1776,graylog_warm_1654,graylog_warm_1775,graylog_warm_1657,graylog_warm_1778,graylog_warm_1656,graylog_warm_1777,graylog_warm_1659,graylog_warm_1658,graylog_warm_1779,graylog_warm_1790,graylog_warm_1671,graylog_warm_1792,graylog_warm_1670,graylog_warm_1791,graylog_warm_1673,graylog_warm_1794,graylog_warm_1672,graylog_warm_1793,graylog_warm_1675,graylog_warm_1796,graylog_warm_1674,graylog_warm_1795,graylog_warm_1666,graylog_warm_1787,graylog_warm_1665,graylog_warm_1786,graylog_warm_1668,graylog_warm_1789,graylog_warm_1667,graylog_warm_1788,graylog_warm_1669,graylog_warm_1640,graylog_warm_1761,graylog_warm_1760,graylog_warm_1642,graylog_warm_1763,graylog_warm_1641,graylog_warm_1762,graylog_warm_1754,graylog_warm_1753,graylog_warm_1756,graylog_warm_1755,graylog_warm_1758,graylog_warm_1757,graylog_warm_1759,gl_windows_sysmon_777,gl_windows_sysmon_778,graylog_warm_1770,graylog_warm_1651,graylog_warm_1772,graylog_warm_1650,graylog_warm_1771,graylog_warm_1653,graylog_warm_1774,graylog_warm_1652,graylog_warm_1773,graylog_warm_1644,graylog_warm_1765,graylog_warm_1643,graylog_warm_1764,graylog_warm_1646,graylog_warm_1767,graylog_warm_1645,graylog_warm_1766,graylog_warm_1648,graylog_warm_1769,graylog_warm_1647,graylog_warm_1768,graylog_warm_1649/_search?typed_keys=true&max_concurrent_shard_requests=5&ignore_unavailable=true&expand_wildcards=open&allow_no_indices=true&ignore_throttled=false&search_type=query_then_fetch&batched_reduce_size=512&ccs_minimize_roundtrips=true], status line [HTTP/1.0 400 Bad Request]
{"error":{"root_cause":[{"type":"too_long_http_line_exception","reason":"An HTTP line is larger than 4096 bytes."}],"type":"too_long_http_line_exception","reason":"An HTTP line is larger than 4096 bytes."},"status":400}
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestClient.convertResponse(RestClient.java:376) ~[?:?]
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestClient.performRequest(RestClient.java:346) ~[?:?]
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestClient.performRequest(RestClient.java:321) ~[?:?]
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1918) ~[?:?]
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1884) ~[?:?]
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1852) ~[?:?]
		at org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1095) ~[?:?]
		at org.graylog.storage.opensearch2.PaginationResultOS2.lambda$nextSearchResult$0(PaginationResultOS2.java:56) ~[?:?]
		at org.graylog.storage.opensearch2.OpenSearchClient.executeWithIOException(OpenSearchClient.java:160) ~[?:?]
		at org.graylog.storage.opensearch2.PaginationResultOS2.nextSearchResult(PaginationResultOS2.java:56) ~[?:?]
		at org.graylog.storage.opensearch2.PaginationResultOS2.nextSearchResult(PaginationResultOS2.java:31) ~[?:?]
		at org.graylog2.indexer.results.ChunkedQueryResult.nextChunk(ChunkedQueryResult.java:68) ~[graylog.jar:?]
		at org.graylog.storage.opensearch2.MoreSearchAdapterOS2.scrollEvents(MoreSearchAdapterOS2.java:170) ~[?:?]
		at org.graylog.events.search.MoreSearch.scrollQuery(MoreSearch.java:157) ~[graylog.jar:?]
		at org.graylog.events.processor.aggregation.AggregationEventProcessor.filterSearch(AggregationEventProcessor.java:285) ~[graylog.jar:?]
		at org.graylog.events.processor.aggregation.AggregationEventProcessor.createEvents(AggregationEventProcessor.java:146) ~[graylog.jar:?]
		at org.graylog.events.processor.EventProcessorEngine.execute(EventProcessorEngine.java:100) ~[graylog.jar:?]
		at org.graylog.events.processor.EventProcessorExecutionJob.execute(EventProcessorExecutionJob.java:115) ~[graylog.jar:?]
		at org.graylog.scheduler.JobExecutionEngine.executeJob(JobExecutionEngine.java:292) ~[graylog.jar:?]
		at org.graylog.scheduler.JobExecutionEngine.lambda$handleTrigger$4(JobExecutionEngine.java:265) ~[graylog.jar:?]
		at com.codahale.metrics.Timer.time(Timer.java:151) ~[graylog.jar:?]
		at org.graylog.scheduler.JobExecutionEngine.handleTrigger(JobExecutionEngine.java:265) ~[graylog.jar:?]
		at org.graylog.scheduler.JobExecutionEngine.handleTriggerWithConcurrencyLimit(JobExecutionEngine.java:237) ~[graylog.jar:?]
		at org.graylog.scheduler.JobExecutionEngine.lambda$execute$2(JobExecutionEngine.java:202) ~[graylog.jar:?]
		at org.graylog.scheduler.worker.JobWorkerPool.lambda$execute$0(JobWorkerPool.java:115) ~[graylog.jar:?]
		at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:259) [graylog.jar:?]
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
		at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66) [graylog.jar:?]
		at java.base/java.lang.Thread.run(Thread.java:840) [?:?]
```

## Possible Solution
In this case we should either reduce the amount of indices send via url to OpenSearch.
Or warn the user that the amount of indices send via URL is to big and he wants to reduce the timerange to limit the amount of indices.

## Context
Found in at a customer enviorment.




Discussion / hints:
partly fixed by https://github.com/Graylog2/graylog2-server/pull/21592
(this PR should reduce the number of indices queried significantly)

Fix patch:
diff --git a/changelog/unreleased/pr-22162.toml b/changelog/unreleased/pr-22162.toml
new file mode 100644
index 000000000000..fc09219bdfa6
--- /dev/null
+++ b/changelog/unreleased/pr-22162.toml
@@ -0,0 +1,5 @@
+type = "fixed" # One of: a(dded), c(hanged), d(eprecated), r(emoved), f(ixed), s(ecurity)
+message = "Use msearch instead of search to avoid creating too long HTTP request lines for getting the next chunk in pagination calls"
+
+issues = ["21554"]
+pulls = ["22162"]
diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/PaginationResultES7.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/PaginationResultES7.java
index efac410aee2d..8d48a1e340bf 100644
--- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/PaginationResultES7.java
+++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/PaginationResultES7.java
@@ -45,7 +45,7 @@ public PaginationResultES7(ResultMessageFactory resultMessageFactory,
 
     @Override
     @Nullable
-    protected SearchResponse nextSearchResult() throws IOException {
+    protected SearchResponse nextSearchResult() {
         final SearchSourceBuilder initialQuery = initialSearchRequest.source();
         final SearchHit[] hits = lastSearchResponse.getHits().getHits();
         if (hits == null || hits.length == 0) {
@@ -53,8 +53,7 @@ protected SearchResponse nextSearchResult() throws IOException {
         }
         initialQuery.searchAfter(hits[hits.length - 1].getSortValues());
         initialSearchRequest.source(initialQuery);
-        return client.executeWithIOException((c, requestOptions) -> c.search(initialSearchRequest, requestOptions),
-                "Unable to retrieve next chunk from search: ");
+        return client.search(initialSearchRequest, "Unable to retrieve next chunk from search: ");
     }
 
     @Override
diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/PaginationResultOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/PaginationResultOS2.java
index 62d98bed4249..df6b36d1ac0e 100644
--- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/PaginationResultOS2.java
+++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/PaginationResultOS2.java
@@ -45,7 +45,7 @@ public PaginationResultOS2(ResultMessageFactory resultMessageFactory,
 
     @Override
     @Nullable
-    protected SearchResponse nextSearchResult() throws IOException {
+    protected SearchResponse nextSearchResult() {
         final SearchSourceBuilder initialQuery = initialSearchRequest.source();
         final SearchHit[] hits = lastSearchResponse.getHits().getHits();
         if (hits == null || hits.length == 0) {
@@ -53,8 +53,7 @@ protected SearchResponse nextSearchResult() throws IOException {
         }
         initialQuery.searchAfter(hits[hits.length - 1].getSortValues());
         initialSearchRequest.source(initialQuery);
-        return client.executeWithIOException((c, requestOptions) -> c.search(initialSearchRequest, requestOptions),
-                "Unable to retrieve next chunk from search: ");
+        return client.search(initialSearchRequest, "Unable to retrieve next chunk from search: ");
     }
 
     @Override


Test patch: