Repository: parseablehq/parseable
Pull request #1236: feat: prism post datasets API
Author: login: de-sh
Labels: name: for next release
Created: 2025-03-13T11:18:21Z | Merged: 2025-03-18T06:52:11Z

<!-- Thanks for trying to help us make Parseable be the best it can be! Please fill out as much of the following information as is possible (where relevant, and remove it when irrelevant) to help make the intention and scope of this PR clear in order to ease review. -->

Fixes #XXXX.

<!-- Replace XXXX with the id of the issue fixed in this PR. Remove this section if there is no corresponding issue. Don't reference the issue in the title of this pull-request. -->

### Description

### Example
Request:
```js
{"streams": ["test"]}
```

Response:
```json
[
  {
    "stream": "test",
    "info": {
      "created-at": "2025-03-14T18:41:15.342294+00:00",
      "first-event-at": "2025-03-15T00:11:15.344+05:30",
      "stream_type": "UserDefined",
      "log_source": [
        {
          "log_source_format": "Json",
          "fields": []
        }
      ]
    },
    "stats": {
      "stream": "test",
      "time": "2025-03-17T09:15:41.884455Z",
      "ingestion": {
        "count": 38452150,
        "size": 17461745043,
        "format": "json",
        "lifetime_count": 38452150,
        "lifetime_size": 17461745043,
        "deleted_count": 0,
        "deleted_size": 0
      },
      "storage": {
        "size": 5078719817,
        "format": "parquet",
        "lifetime_size": 5078719817,
        "deleted_size": 0
      }
    },
    "retention": [],
    "hottier": null,
    "counts": {
      "fields": [
        "start_time",
        "end_time",
        "count"
      ],
      "records": [
        {
          "start_time": "2025-03-17T08:15:41.884559+00:00",
          "end_time": "2025-03-17T08:21:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:21:41.884559+00:00",
          "end_time": "2025-03-17T08:27:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:27:41.884559+00:00",
          "end_time": "2025-03-17T08:33:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:33:41.884559+00:00",
          "end_time": "2025-03-17T08:39:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:39:41.884559+00:00",
          "end_time": "2025-03-17T08:45:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:45:41.884559+00:00",
          "end_time": "2025-03-17T08:51:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:51:41.884559+00:00",
          "end_time": "2025-03-17T08:57:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T08:57:41.884559+00:00",
          "end_time": "2025-03-17T09:03:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T09:03:41.884559+00:00",
          "end_time": "2025-03-17T09:09:41.884559+00:00",
          "count": 0
        },
        {
          "start_time": "2025-03-17T09:09:41.884559+00:00",
          "end_time": "2025-03-17T09:15:41.884559+00:00",
          "count": 0
        }
      ]
    },
    "distinct_sources": {
      "ips": null,
      "user_agents": null
    }
  }
]
```

NOTE: returns for all streams when none are mentioned or body is empty

<hr>

This PR has:
- [ ] been tested to ensure log ingestion and log query works.
- [ ] added comments explaining the "why" and the intent of the code wherever would not be obvious for an unfamiliar reader.
- [ ] added documentation for new or modified features or behaviors.


<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

## Summary by CodeRabbit

- **New Features**
  - Introduced a new endpoint for dataset queries, enabling secure POST requests to retrieve comprehensive dataset details including statistics, retention information, and record counts.
  
- **Refactor**
  - Streamlined data processing and response formatting to improve performance and reliability when handling dataset requests.
  - Simplified the conversion of record batches to JSON, enhancing code efficiency.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->


Diff:
diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index c51d98bd4..161bd88f5 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -76,7 +76,8 @@ impl ParseableServer for QueryServer {
             .service(
                 web::scope(&prism_base_path())
                     .service(Server::get_prism_home())
-                    .service(Server::get_prism_logstream()),
+                    .service(Server::get_prism_logstream())
+                    .service(Server::get_prism_datasets()),
             )
             .service(Server::get_generated());
     }
diff --git a/src/handlers/http/modal/server.rs b/src/handlers/http/modal/server.rs
index 38d0e0239..d28b02b59 100644
--- a/src/handlers/http/modal/server.rs
+++ b/src/handlers/http/modal/server.rs
@@ -95,7 +95,8 @@ impl ParseableServer for Server {
             .service(
                 web::scope(&prism_base_path())
                     .service(Server::get_prism_home())
-                    .service(Server::get_prism_logstream()),
+                    .service(Server::get_prism_logstream())
+                    .service(Server::get_prism_datasets()),
             )
             .service(Self::get_ingest_otel_factory())
             .service(Self::get_generated());
@@ -180,6 +181,17 @@ impl Server {
         )
     }
 
+    pub fn get_prism_datasets() -> Scope {
+        web::scope("/datasets").route(
+            "",
+            web::post()
+                .to(http::prism_logstream::post_datasets)
+                .authorize_for_stream(Action::GetStreamInfo)
+                .authorize_for_stream(Action::GetStats)
+                .authorize_for_stream(Action::GetRetention),
+        )
+    }
+
     pub fn get_metrics_webscope() -> Scope {
         web::scope("/metrics").service(
             web::resource("").route(web::get().to(metrics::get).authorize(Action::Metrics)),
diff --git a/src/handlers/http/prism_logstream.rs b/src/handlers/http/prism_logstream.rs
index 2be76a094..69b59f5a5 100644
--- a/src/handlers/http/prism_logstream.rs
+++ b/src/handlers/http/prism_logstream.rs
@@ -17,11 +17,14 @@
  */
 
 use actix_web::{
-    web::{self, Path},
-    Responder,
+    web::{self, Json, Path},
+    HttpRequest, Responder,
 };
 
-use crate::prism::logstream::{get_prism_logstream_info, PrismLogstreamError};
+use crate::{
+    prism::logstream::{get_prism_logstream_info, PrismDatasetRequest, PrismLogstreamError},
+    utils::actix::extract_session_key_from_req,
+};
 
 /// This API is essentially just combining the responses of /info and /schema together
 pub async fn get_info(stream_name: Path<String>) -> Result<impl Responder, PrismLogstreamError> {
@@ -29,3 +32,18 @@ pub async fn get_info(stream_name: Path<String>) -> Result<impl Responder, Prism
 
     Ok(web::Json(prism_logstream_info))
 }
+
+/// A combination of /stats, /retention, /hottier, /info, /counts and /query
+pub async fn post_datasets(
+    dataset_req: Option<Json<PrismDatasetRequest>>,
+    req: HttpRequest,
+) -> Result<impl Responder, PrismLogstreamError> {
+    let session_key = extract_session_key_from_req(&req)?;
+    let dataset = dataset_req
+        .map(|Json(r)| r)
+        .unwrap_or_default()
+        .get_datasets(session_key)
+        .await?;
+
+    Ok(web::Json(dataset))
+}
diff --git a/src/prism/logstream/mod.rs b/src/prism/logstream/mod.rs
index e9ffcb981..e37db20e4 100644
--- a/src/prism/logstream/mod.rs
+++ b/src/prism/logstream/mod.rs
@@ -22,7 +22,9 @@ use actix_web::http::header::ContentType;
 use arrow_schema::Schema;
 use chrono::Utc;
 use http::StatusCode;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+use tracing::{debug, warn};
 
 use crate::{
     handlers::http::{
@@ -31,11 +33,18 @@ use crate::{
             utils::{merge_quried_stats, IngestionStats, QueriedStats, StorageStats},
         },
         logstream::error::StreamError,
-        query::update_schema_when_distributed,
+        query::{into_query, update_schema_when_distributed, Query, QueryError},
     },
+    hottier::{HotTierError, HotTierManager, StreamHotTier},
     parseable::{StreamNotFound, PARSEABLE},
+    query::{error::ExecuteError, execute, CountsRequest, CountsResponse, QUERY_SESSION},
+    rbac::{map::SessionKey, role::Action, Users},
     stats,
     storage::{retention::Retention, StreamInfo, StreamType},
+    utils::{
+        arrow::record_batches_to_json,
+        time::{TimeParseError, TimeRange},
+    },
     LOCK_EXPECT,
 };
 
@@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Name of the stream
+    stream: String,
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query
+#[derive(Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    ///    successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        let is_empty = self.streams.is_empty();
+        if is_empty {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                // Don't warn if listed from Parseable
+                if !is_empty {
+                    warn!("Unauthorized access requested for stream: {stream}");
+                }
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: "1h".to_owned(),
+                end_time: "now".to_owned(),
+                num_bins: 10,
+            }
+            .get_bin_density()
+            .await?;
+            let counts = CountsResponse {
+                fields: vec!["start_time".into(), "end_time".into(), "count".into()],
+                records,
+            };
+
+            // Retrieve distinct values for source identifiers
+            // Returns None if fields aren't present or if query fails
+            let ips = self.get_distinct_entries(stream, "p_src_ip").await.ok();
+            let user_agents = self.get_distinct_entries(stream, "p_user_agent").await.ok();
+
+            responses.push(PrismDatasetResponse {
+                stream: stream.clone(),
+                info,
+                stats,
+                retention,
+                hottier,
+                counts,
+                distinct_sources: json!({
+                    "ips": ips,
+                    "user_agents": user_agents
+                }),
+            })
+        }
+
+        Ok(responses)
+    }
+
+    /// Retrieves distinct values for a specific field in a stream.
+    ///
+    /// # Parameters
+    /// - `stream_name`: Name of the stream to query
+    /// - `field`: Field name to get distinct values for
+    ///
+    /// # Returns
+    /// - `Ok(Vec<String>)`: List of distinct values found for the field
+    /// - `Err(QueryError)`: If the query fails or field doesn't exist
+    async fn get_distinct_entries(
+        &self,
+        stream_name: &str,
+        field: &str,
+    ) -> Result<Vec<String>, QueryError> {
+        let query = Query {
+            query: format!("SELECT DISTINCT({field}) FOR {stream_name}"),
+            start_time: "1h".to_owned(),
+            end_time: "now".to_owned(),
+            send_null: false,
+            filter_tags: None,
+            fields: true,
+        };
+        let time_range = TimeRange::parse_human_time("1h", "now")?;
+
+        let session_state = QUERY_SESSION.state();
+        let query = into_query(&query, &session_state, time_range).await?;
+        let (records, _) = execute(query, stream_name).await?;
+        let response = record_batches_to_json(&records)?;
+        // Extract field values from the JSON response
+        let values = response
+            .iter()
+            .flat_map(|row| {
+                row.get(field)
+                    .and_then(|s| s.as_str())
+                    .map(|s| s.to_string())
+            })
+            .collect();
+
+        Ok(values)
+    }
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum PrismLogstreamError {
     #[error("Error: {0}")]
@@ -193,6 +364,16 @@ pub enum PrismLogstreamError {
     StreamError(#[from] StreamError),
     #[error("StreamNotFound: {0}")]
     StreamNotFound(#[from] StreamNotFound),
+    #[error("Hottier: {0}")]
+    Hottier(#[from] HotTierError),
+    #[error("Query: {0}")]
+    Query(#[from] QueryError),
+    #[error("TimeParse: {0}")]
+    TimeParse(#[from] TimeParseError),
+    #[error("Execute: {0}")]
+    Execute(#[from] ExecuteError),
+    #[error("Auth: {0}")]
+    Auth(#[from] actix_web::Error),
 }
 
 impl actix_web::ResponseError for PrismLogstreamError {
@@ -201,6 +382,11 @@ impl actix_web::ResponseError for PrismLogstreamError {
             PrismLogstreamError::Anyhow(_) => StatusCode::INTERNAL_SERVER_ERROR,
             PrismLogstreamError::StreamError(e) => e.status_code(),
             PrismLogstreamError::StreamNotFound(_) => StatusCode::NOT_FOUND,
+            PrismLogstreamError::Hottier(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            PrismLogstreamError::Query(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            PrismLogstreamError::TimeParse(_) => StatusCode::NOT_FOUND,
+            PrismLogstreamError::Execute(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            PrismLogstreamError::Auth(_) => StatusCode::UNAUTHORIZED,
         }
     }
 
diff --git a/src/response.rs b/src/response.rs
index 8211bf4c7..fe3159f71 100644
--- a/src/response.rs
+++ b/src/response.rs
@@ -33,8 +33,7 @@ pub struct QueryResponse {
 impl QueryResponse {
     pub fn to_http(&self) -> Result<HttpResponse, QueryError> {
         info!("{}", "Returning query results");
-        let records: Vec<&RecordBatch> = self.records.iter().collect();
-        let mut json_records = record_batches_to_json(&records)?;
+        let mut json_records = record_batches_to_json(&self.records)?;
 
         if self.fill_null {
             for map in &mut json_records {
diff --git a/src/utils/arrow/mod.rs b/src/utils/arrow/mod.rs
index 53e6437d6..e29762047 100644
--- a/src/utils/arrow/mod.rs
+++ b/src/utils/arrow/mod.rs
@@ -90,10 +90,12 @@ pub fn replace_columns(
 /// * Result<Vec<Map<String, Value>>>
 ///
 /// A vector of JSON objects representing the record batches.
-pub fn record_batches_to_json(records: &[&RecordBatch]) -> Result<Vec<Map<String, Value>>> {
+pub fn record_batches_to_json(records: &[RecordBatch]) -> Result<Vec<Map<String, Value>>> {
     let buf = vec![];
     let mut writer = arrow_json::ArrayWriter::new(buf);
-    writer.write_batches(records)?;
+    for record in records {
+        writer.write(record)?;
+    }
     writer.finish()?;
 
     let buf = writer.into_inner();
@@ -188,7 +190,7 @@ mod tests {
     #[test]
     fn check_empty_json_to_record_batches() {
         let r = RecordBatch::new_empty(Arc::new(Schema::empty()));
-        let rb = vec![&r];
+        let rb = vec![r];
         let batches = record_batches_to_json(&rb).unwrap();
         assert_eq!(batches, vec![]);
     }


Reviews:
user: login: coderabbitai[bot]
body: **Actionable comments posted: 0**

<details>
<summary>🧹 Nitpick comments (2)</summary><blockquote>

<details>
<summary>src/prism/logstream/mod.rs (2)</summary><blockquote>

`196-213`: **Well-documented dataset response structure.**

`PrismDatasetResponse` encapsulates essential stream metadata, statistics, retention, and query results. The doc comments are thorough. Consider storing `distinct_sources` in a typed struct if the shape is known, but using `serde_json::Value` is acceptable for flexibility.

---

`229-345`: **Robust dataset retrieval and aggregation logic.**

1. The loop processes each stream, skipping unavailable ones, which strengthens resilience.
2. Hot tier information and distinct entry queries are well-integrated.
3. Swallowing errors in distinct lookups is suitable for partial success, but consider logging warnings on failure for better observability.
4. Parallelizing the loop (using tasks or rayon) might improve performance for large stream lists, but the current sequential approach is simpler to maintain.

Overall, the code is well-structured, and the error handling strategy aligns with partial success semantics.

</blockquote></details>

</blockquote></details>

<details>
<summary>📜 Review details</summary>

**Configuration used: CodeRabbit UI**
**Review profile: CHILL**
**Plan: Pro**

<details>
<summary>📥 Commits</summary>

Reviewing files that changed from the base of the PR and between 506527d566898f61a42f6aa4347eafba54dec69a and 536782038946b6ec19283e38c8e1dcd9da8d6b8e.

</details>

<details>
<summary>📒 Files selected for processing (6)</summary>

* `src/handlers/http/modal/query_server.rs` (1 hunks)
* `src/handlers/http/modal/server.rs` (2 hunks)
* `src/handlers/http/prism_logstream.rs` (1 hunks)
* `src/prism/logstream/mod.rs` (5 hunks)
* `src/response.rs` (1 hunks)
* `src/utils/arrow/mod.rs` (2 hunks)

</details>

<details>
<summary>⏰ Context from checks skipped due to timeout of 90000ms (10)</summary>

* GitHub Check: Quest Smoke and Load Tests for Distributed deployments
* GitHub Check: Quest Smoke and Load Tests for Standalone deployments
* GitHub Check: Build Default x86_64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-pc-windows-msvc
* GitHub Check: coverage
* GitHub Check: Build Default x86_64-apple-darwin
* GitHub Check: Build Default aarch64-apple-darwin
* GitHub Check: Build Default aarch64-unknown-linux-gnu
* GitHub Check: Build Kafka aarch64-apple-darwin
* GitHub Check: Build Kafka x86_64-unknown-linux-gnu

</details>

<details>
<summary>🔇 Additional comments (23)</summary><blockquote>

<details>
<summary>src/handlers/http/modal/query_server.rs (1)</summary>

`79-80`: **Looks good! Added prism datasets service.**

The addition of the datasets service aligns well with the existing infrastructure and properly follows the established pattern next to the logstream service.

</details>
<details>
<summary>src/handlers/http/modal/server.rs (1)</summary>

`184-193`: **Implementation of datasets endpoint follows best practices.**

The implementation follows the established pattern for route declaration and properly includes the necessary authorization checks for stream info, stats, and retention.

</details>
<details>
<summary>src/response.rs (1)</summary>

`36-36`: **Good refactoring to simplify JSON conversion.**

This change removes an unnecessary intermediate step, directly passing the record batches to the conversion function. This simplifies the code while maintaining the same functionality.

</details>
<details>
<summary>src/handlers/http/prism_logstream.rs (3)</summary>

`20-20`: **Updated imports to support JSON handling.**

Added the Json type import to support parsing request bodies for the new datasets endpoint.

---

`24-24`: **Added necessary imports for dataset functionality.**

Included PrismDatasetRequest to support the new datasets endpoint functionality.

---

`33-38`: **Well-implemented handler for the datasets endpoint.**

This handler properly:
1. Takes a JSON payload via the `Json` extractor
2. Processes the request asynchronously 
3. Returns a properly formatted JSON response
4. Includes appropriate error handling

The comment also clearly explains that this endpoint combines functionality from multiple other endpoints.

</details>
<details>
<summary>src/utils/arrow/mod.rs (3)</summary>

`93-93`: **Refactored function signature for enhanced clarity.**

Switching to a slice of owned `RecordBatch` objects (`&[RecordBatch]`) ensures a more standard API and simplifies usage, preventing potential reference lifetime pitfalls.

---

`96-98`: **Looping over batches is straightforward and concise.**

Writing each `RecordBatch` in a loop is a clear and maintainable approach. The usage of the `?` operator for error propagation is consistent with idiomatic Rust error handling.

---

`194-194`: **Consistent with the updated function signature.**

Switching from `vec![&r]` to `vec![r]` matches the new function parameter style. This change eliminates the need for references to `RecordBatch` in this context.

</details>
<details>
<summary>src/prism/logstream/mod.rs (14)</summary>

`25-27`: **New imports for serialization, JSON handling, and debugging.**

Adding `serde`, `serde_json`, and `tracing` imports is appropriate for the newly introduced dataset and logging features.

---

`36-36`: **Extended query imports.**

Introducing `Query` and `QueryError` in the imports indicates new query functionalities specific to this module, streamlining error handling and query building.

---

`38-38`: **Hot Tier Manager integration.**

Importing `HotTierError`, `HotTierManager`, and `StreamHotTier` suggests advanced stream management and a new dimension of error handling for hot tier capabilities.

---

`40-40`: **Expanded query utilities.**

Bringing in `execute`, `CountsRequest`, `CountsResponse`, and `QUERY_SESSION` aligns with your new approach to handle count queries within the streaming logic.

---

`43-46`: **Utility imports for record batch JSON conversion and time parsing.**

Using `record_batches_to_json` and `TimeRange` helps unify record transformations and date handling, improving modularity and readability across the codebase.

---

`215-227`: **Clear request structure for dataset queries.**

`PrismDatasetRequest` neatly defines the data needed to retrieve datasets. The user-friendly doc comments help clarify field usage. Ensure upstream validation rejects invalid or empty time strings if needed.

---

`355-355`: **New error annotation for hot tier.**

`#[error("Hottier: {0}")]` extends clarity by rendering a descriptive message when a hot tier-related error occurs.

---

`356-357`: **Introducing Hottier variant with from-attribute.**

`Hottier(#[from] HotTierError)` and `Query(#[from] QueryError)` make error handling consistent and concise through `thiserror`.

---

`359-359`: **Readable time parse error message.**

`#[error("TimeParse: {0}")]` ensures that users can quickly identify problems with time string inputs.

---

`360-361`: **Time parsing and execute error variants.**

`TimeParse(#[from] TimeParseError)` and `#[error("Execute: {0}")]` unify typical parse and execution failures within the same error enumeration.

---

`363-364`: **Empty stream list checks.**

Returning a `BAD_REQUEST` (see below) for an empty stream list is a sensible choice, guiding the user to provide at least one stream.

---

`373-373`: **Mapping hot tier errors to 500.**

`PrismLogstreamError::Hottier(_) => StatusCode::INTERNAL_SERVER_ERROR` is consistent with critical server-side failure logic.

---

`374-375`: **Query and time parse error statuses.**

1. Query errors remain 500, indicating back-end failures.  
2. Time parse errors map to 404, guiding the user to correct the time string usage.

---

`377-377`: **Empty request yields a 400 Bad Request.**

Returning `StatusCode::BAD_REQUEST` helps users recognize that their request must contain valid streams.

</details>

</blockquote></details>

</details>

<!-- This is an auto-generated comment by CodeRabbit for review status -->
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:22:07Z
user: login: coderabbitai[bot]
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:22:10Z
user: login: coderabbitai[bot]
body: **Actionable comments posted: 1**

<details>
<summary>🧹 Nitpick comments (1)</summary><blockquote>

<details>
<summary>src/prism/logstream/mod.rs (1)</summary><blockquote>

`244-315`: **Consider parallel processing for multiple streams.**

For requests with many streams, processing each stream sequentially might lead to increased response times. Consider using parallel processing with `futures::future::join_all` to improve performance for multi-stream requests.


```diff
-        let mut responses = vec![];
-        for stream in self.streams.iter() {
-            // existing processing logic
-        }
+        // Create a vector of futures for each stream
+        let futures = self.streams.iter().map(|stream| {
+            let stream = stream.clone();
+            let start_time = self.start_time.clone();
+            let end_time = self.end_time.clone();
+            let key = key.clone();
+            
+            async move {
+                // Skip if unauthorized or stream not found
+                if Users.authorize(key, Action::ListStream, Some(&stream), None)
+                    != crate::rbac::Response::Authorized
+                {
+                    warn!("Unauthorized access requested for stream: {stream}");
+                    return None;
+                }
+                
+                if PARSEABLE.check_or_load_stream(&stream).await {
+                    debug!("Stream not found: {stream}");
+                    return None;
+                }
+                
+                // Process stream (existing logic)
+                match process_single_stream(&stream, &start_time, &end_time).await {
+                    Ok(response) => Some(response),
+                    Err(e) => {
+                        warn!("Error processing stream {stream}: {e}");
+                        None
+                    }
+                }
+            }
+        });
+        
+        // Process all streams in parallel
+        let responses: Vec<_> = futures::future::join_all(futures)
+            .await
+            .into_iter()
+            .filter_map(|result| result)
+            .collect();
```

You would need to extract the stream processing logic into a separate `process_single_stream` function.

</blockquote></details>

</blockquote></details>

<details>
<summary>📜 Review details</summary>

**Configuration used: CodeRabbit UI**
**Review profile: CHILL**
**Plan: Pro**

<details>
<summary>📥 Commits</summary>

Reviewing files that changed from the base of the PR and between 536782038946b6ec19283e38c8e1dcd9da8d6b8e and 427316de5ad1b0f7215732534bfe571d5bdcd009.

</details>

<details>
<summary>📒 Files selected for processing (2)</summary>

* `src/handlers/http/prism_logstream.rs` (1 hunks)
* `src/prism/logstream/mod.rs` (5 hunks)

</details>

<details>
<summary>⏰ Context from checks skipped due to timeout of 90000ms (10)</summary>

* GitHub Check: coverage
* GitHub Check: Build Default aarch64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-apple-darwin
* GitHub Check: Build Default x86_64-pc-windows-msvc
* GitHub Check: Build Default aarch64-apple-darwin
* GitHub Check: Build Kafka x86_64-unknown-linux-gnu
* GitHub Check: Build Kafka aarch64-apple-darwin
* GitHub Check: Quest Smoke and Load Tests for Distributed deployments
* GitHub Check: Quest Smoke and Load Tests for Standalone deployments

</details>

<details>
<summary>🔇 Additional comments (5)</summary><blockquote>

<details>
<summary>src/handlers/http/prism_logstream.rs (1)</summary>

`32-38`: **Function implementation looks good.**

The new `post_datasets` function correctly extracts the session key from the request, processes the dataset query, and returns a JSON response. The implementation follows the established pattern in the codebase.

</details>
<details>
<summary>src/prism/logstream/mod.rs (4)</summary>

`197-214`: **Well-structured response model with comprehensive documentation.**

The `PrismDatasetResponse` struct is appropriately designed to encapsulate all relevant information about a dataset, including stream info, stats, retention policy, and query results. The documentation clearly explains each field's purpose.

---

`216-228`: **Request model follows good design practices.**

The `PrismDatasetRequest` struct properly uses Rust idioms like `#[serde(rename_all = "camelCase")]` for JSON serialization/deserialization and provides clear field documentation.

---

`367-376`: **Good error handling extension.**

The additions to the `PrismLogstreamError` enum appropriately cover all the new error cases that might occur during dataset operations. This ensures proper error propagation and handling throughout the system.

---

`385-389`: **Appropriate HTTP status codes for error types.**

The status code mapping for the new error variants is appropriate, with authentication errors returning 401, time parsing errors returning 404, and internal processing errors returning 500.

</details>

</blockquote></details>

</details>

<!-- This is an auto-generated comment by CodeRabbit for review status -->
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:37:37Z
user: login: de-sh
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:39:29Z
user: login: coderabbitai[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:39:53Z
user: login: coderabbitai[bot]
body: **Actionable comments posted: 0**

<details>
<summary>🧹 Nitpick comments (1)</summary><blockquote>

<details>
<summary>src/handlers/http/prism_logstream.rs (1)</summary><blockquote>

`36-45`: **New endpoint implementation for dataset retrieval.**

The new `post_datasets` function is well-structured and follows established patterns:
1. It extracts the session key for authentication
2. Passes the session key to the dataset retrieval method
3. Returns the result as JSON

The function correctly uses async/await with proper error propagation through the `?` operator.

Consider adding more specific documentation about:
- The expected format of the request payload
- The response structure
- Any authorization requirements


Enhance the function documentation to be more specific about request/response format:

```diff
-/// A combination of /stats, /retention, /hottier, /info, /counts and /query
+/// Retrieves dataset information for a Prism logstream
+/// 
+/// Takes a JSON payload with stream identifier and time range, and returns
+/// dataset details including creation time, stream type, log source, and
+/// various statistics related to ingestion and storage.
+///
+/// Requires authentication via session key.
```

</blockquote></details>

</blockquote></details>

<details>
<summary>📜 Review details</summary>

**Configuration used: CodeRabbit UI**
**Review profile: CHILL**
**Plan: Pro**

<details>
<summary>📥 Commits</summary>

Reviewing files that changed from the base of the PR and between 427316de5ad1b0f7215732534bfe571d5bdcd009 and 90df5929d3a460434a8dd40f27643e17102a61d1.

</details>

<details>
<summary>📒 Files selected for processing (1)</summary>

* `src/handlers/http/prism_logstream.rs` (1 hunks)

</details>

<details>
<summary>⏰ Context from checks skipped due to timeout of 90000ms (10)</summary>

* GitHub Check: Build Kafka x86_64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-apple-darwin
* GitHub Check: Build Default aarch64-unknown-linux-gnu
* GitHub Check: Build Default aarch64-apple-darwin
* GitHub Check: Build Default x86_64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-pc-windows-msvc
* GitHub Check: Build Kafka aarch64-apple-darwin
* GitHub Check: Quest Smoke and Load Tests for Standalone deployments
* GitHub Check: Quest Smoke and Load Tests for Distributed deployments
* GitHub Check: coverage

</details>

<details>
<summary>🔇 Additional comments (4)</summary><blockquote>

<details>
<summary>src/handlers/http/prism_logstream.rs (4)</summary>

`19-22`: **Added HttpRequest and Json imports for new functionality.**

The imports are correctly added to support the new `post_datasets` function, providing the necessary components for handling JSON payloads and accessing HTTP request details.

---

`24-27`: **Updated imports to include PrismDatasetRequest type.**

The imports are properly organized to include the new `PrismDatasetRequest` type and the session key extraction utility, maintaining good code organization.

---

`37-40`: **Function signature follows best practices.**

The function signature correctly uses Actix's JSON extractor pattern for the request body and includes the HttpRequest parameter for session key extraction.

---

`41-44`: <details>
<summary>❓ Verification inconclusive</summary>

**Verify authorization checks in the get_datasets implementation.**

The session key extraction and passing looks good. However, this code doesn't show the implementation of the actual authorization check that happens within `get_datasets`.

---

🏁 Script executed:

```shell
#!/bin/bash
# Check if there are proper authorization checks in the get_datasets implementation
rg -A 10 "fn get_datasets.*session_key" --type rust
```

Length of output: 53

---



</details>

**ACTION: Additional authorization verification required via manual or extended regex search**

The session key extraction logic is solid, but our initial regex search did not locate the `get_datasets` implementation for verifying its internal authorization checks. Please investigate further by:

- Locating the full definition of `fn get_datasets` to inspect for any authorization-related logic.
- Checking for keywords like `authorize`, `permission`, or `auth` in the function’s implementation.

For example, run this extended shell script to search all Rust files for the definition and check for authorization check patterns:

```shell
#!/bin/bash
# Find all occurrences of the get_datasets function and inspect following lines for authorization logic
rg "fn get_datasets" --type rust

# For each file found, display the function's context and look for authorization keywords
for file in $(rg -l "fn get_datasets" --type rust); do
    echo "Inspecting file: $file"
    rg -A 15 "fn get_datasets" "$file" | grep -iE "authorize|permission|auth"
done
```

Please verify the output manually to ensure that appropriate authorization checks are implemented.

</details>

</blockquote></details>

</details>

<!-- This is an auto-generated comment by CodeRabbit for review status -->
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:44:43Z
user: login: coderabbitai[bot]
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:53:13Z
user: login: coderabbitai[bot]
body: **Actionable comments posted: 0**

<details>
<summary>🧹 Nitpick comments (1)</summary><blockquote>

<details>
<summary>src/prism/logstream/mod.rs (1)</summary><blockquote>

`226-312`: **Consider making time ranges configurable in dataset retrieval.**

The implementation is robust with good error handling and authorization checks. However, the time ranges for `CountsRequest` are hardcoded to "1h" and "now". Consider making these configurable through the API request to improve flexibility.

```diff
 #[derive(Deserialize, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct PrismDatasetRequest {
     /// List of stream names to query
     #[serde(default)]
     streams: Vec<String>,
+    /// Start time for queries (default: "1h")
+    #[serde(default = "default_start_time")]
+    start_time: String,
+    /// End time for queries (default: "now")
+    #[serde(default = "default_end_time")]
+    end_time: String,
 }
+
+fn default_start_time() -> String {
+    "1h".to_string()
+}
+
+fn default_end_time() -> String {
+    "now".to_string()
+}
```

Then update the relevant parts of the implementation to use these values:

```diff
         let records = CountsRequest {
             stream: stream.clone(),
-            start_time: "1h".to_owned(),
-            end_time: "now".to_owned(),
+            start_time: self.start_time.clone(),
+            end_time: self.end_time.clone(),
             num_bins: 1,
         }
```

</blockquote></details>

</blockquote></details>

<details>
<summary>📜 Review details</summary>

**Configuration used: CodeRabbit UI**
**Review profile: CHILL**
**Plan: Pro**

<details>
<summary>📥 Commits</summary>

Reviewing files that changed from the base of the PR and between 35561e990a3e453ac5337cc50a884011c1a363d1 and d64b8a84f03f6fd810ce311284b72d24a03758d9.

</details>

<details>
<summary>📒 Files selected for processing (2)</summary>

* `src/handlers/http/prism_logstream.rs` (1 hunks)
* `src/prism/logstream/mod.rs` (5 hunks)

</details>

<details>
<summary>🧰 Additional context used</summary>

<details>
<summary>🧠 Learnings (1)</summary>

<details>
<summary>src/prism/logstream/mod.rs (1)</summary>

```
Learnt from: de-sh
PR: parseablehq/parseable#1236
File: src/prism/logstream/mod.rs:332-332
Timestamp: 2025-03-13T11:39:52.587Z
Learning: SQL injection concerns can be ignored in this codebase as all SQL queries are run against immutable data streams, limiting the potential impact of any injection.
```

</details>

</details>

</details>

<details>
<summary>⏰ Context from checks skipped due to timeout of 90000ms (10)</summary>

* GitHub Check: Build Default x86_64-pc-windows-msvc
* GitHub Check: Build Kafka aarch64-apple-darwin
* GitHub Check: Build Default aarch64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-apple-darwin
* GitHub Check: Build Default aarch64-apple-darwin
* GitHub Check: Build Default x86_64-unknown-linux-gnu
* GitHub Check: Quest Smoke and Load Tests for Standalone deployments
* GitHub Check: Quest Smoke and Load Tests for Distributed deployments
* GitHub Check: Build Kafka x86_64-unknown-linux-gnu
* GitHub Check: coverage

</details>

<details>
<summary>🔇 Additional comments (8)</summary><blockquote>

<details>
<summary>src/handlers/http/prism_logstream.rs (3)</summary>

`20-21`: **New imports support dataset API endpoint functionality.**

The additional imports (`Json`, `HttpRequest`) facilitate the JSON parsing and request handling capabilities needed for the new dataset POST endpoint.

---

`24-27`: **Module imports updated to include dataset-related components.**

The imports correctly include the new `PrismDatasetRequest` type and reuse the existing authorization utility function.

---

`36-49`: **Well-implemented dataset POST handler with clear error handling.**

The implementation follows established patterns for Actix Web handlers:
1. Properly extracts session key for authorization
2. Handles optional JSON payload with sensible defaults
3. Delegates dataset retrieval logic to the model layer
4. Returns a properly formatted JSON response

This maintains separation of concerns between HTTP handling and business logic.

</details>
<details>
<summary>src/prism/logstream/mod.rs (5)</summary>

`197-214`: **Well-structured and documented response type for dataset queries.**

The `PrismDatasetResponse` struct provides a comprehensive representation of dataset information with clear field documentation. The structure follows good API design by grouping related information together.

---

`216-224`: **Request structure uses proper serialization attributes.**

The `PrismDatasetRequest` struct is properly annotated with `#[serde(rename_all = "camelCase")]` for consistent JSON field naming in the API. The `#[serde(default)]` on the `streams` field ensures the API remains backward compatible if the field is omitted.

---

`323-353`: **SQL query construction appears safe in this context.**

The method builds a SQL query by directly interpolating user-provided field names. While this would normally be a SQL injection risk, I understand from the provided context that SQL injection concerns are mitigated in this codebase because queries are run against immutable data streams.

The method effectively retrieves distinct entries and processes the results appropriately. Consider also making the time range configurable here, matching the previous recommendation.

---

`364-373`: **Error types appropriately expanded to cover all failure modes.**

The new error variants in `PrismLogstreamError` properly account for all possible failure modes in the dataset retrieval process. This ensures that errors are accurately reported and can be handled appropriately.

---

`382-386`: **Status codes for new error types are appropriate.**

The mapping of error types to HTTP status codes follows RESTful conventions:
- Internal server errors for processing failures
- Not found for time parsing errors (consistent with resource not found semantics)
- Unauthorized for authentication errors

</details>

</blockquote></details>

</details>

<!-- This is an auto-generated comment by CodeRabbit for review status -->
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:47:16Z
user: login: coderabbitai[bot]
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:47:19Z
user: login: coderabbitai[bot]
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T07:54:49Z
user: login: de-sh
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T09:06:17Z
user: login: de-sh
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T09:07:34Z
user: login: coderabbitai[bot]
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T09:15:36Z
user: login: praveen5959
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T10:15:30Z

Review comments:
diff_hunk: @@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query and the time range for the query.
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+    /// ISO 8601 formatted start time or human-readable time expression
+    start_time: String,
+    /// ISO 8601 formatted end time or human-readable time expression
+    end_time: String,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    /// successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        if self.streams.is_empty() {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                warn!("Unauthorized access requested for stream: {stream}");
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(&stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(&stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(&stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: self.start_time.clone(),
+                end_time: self.end_time.clone(),
+                num_bins: 1,
+            }
+            .get_bin_density()
+            .await?;
+            let counts = CountsResponse {
+                fields: vec!["start_time".into(), "end_time".into(), "count".into()],
+                records,
+            };
+
+            // Retrieve distinct values for source identifiers
+            // Returns None if fields aren't present or if query fails
+            let ips = self.get_distinct_entries(&stream, "p_src_ip").await.ok();
+            let user_agents = self
+                .get_distinct_entries(&stream, "p_user_agent")
+                .await
+                .ok();
+
+            responses.push(PrismDatasetResponse {
+                info,
+                stats,
+                retention,
+                hottier,
+                counts,
+                distinct_sources: json!({
+                    "ips": ips,
+                    "user_agents": user_agents
+                }),
+            })
+        }
+
+        Ok(responses)
+    }
+
+    /// Retrieves distinct values for a specific field in a stream.
+    ///
+    /// # Parameters
+    /// - `stream_name`: Name of the stream to query
+    /// - `field`: Field name to get distinct values for
+    ///
+    /// # Returns
+    /// - `Ok(Vec<String>)`: List of distinct values found for the field
+    /// - `Err(QueryError)`: If the query fails or field doesn't exist
+    async fn get_distinct_entries(
+        &self,
+        stream_name: &str,
+        field: &str,
+    ) -> Result<Vec<String>, QueryError> {
+        let query = Query {
+            query: format!("SELECT DISTINCT({field}) FOR {stream_name}"),
path: src/prism/logstream/mod.rs
user: login: coderabbitai[bot]
body: _⚠️ Potential issue_

**Potential SQL injection risk in query construction.**

The direct string interpolation in the SQL query could be vulnerable to SQL injection if the field name parameter is not properly validated. Consider using parameterized queries or validating field names against a whitelist.


```diff
-            query: format!("SELECT DISTINCT({field}) FOR {stream_name}"),
+            // Validate field name against expected column names
+            query: {
+                // Ensure field name only contains alphanumeric characters and underscores
+                if !field.chars().all(|c| c.is_alphanumeric() || c == '_') {
+                    return Err(QueryError::InvalidField(field.to_string()));
+                }
+                format!("SELECT DISTINCT({field}) FOR {stream_name}")
+            },
```

You may need to add the `InvalidField` variant to the `QueryError` enum if it doesn't exist.


<!-- suggestion_start -->

<details>
<summary>📝 Committable suggestion</summary>

> ‼️ **IMPORTANT**
> Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

`````suggestion
            // Validate field name against expected column names
            query: {
                // Ensure field name only contains alphanumeric characters and underscores
                if !field.chars().all(|c| c.is_alphanumeric() || c == '_') {
                    return Err(QueryError::InvalidField(field.to_string()));
                }
                format!("SELECT DISTINCT({field}) FOR {stream_name}")
            },
`````

</details>

<!-- suggestion_end -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-03-13T11:37:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query and the time range for the query.
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+    /// ISO 8601 formatted start time or human-readable time expression
+    start_time: String,
+    /// ISO 8601 formatted end time or human-readable time expression
+    end_time: String,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    /// successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        if self.streams.is_empty() {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                warn!("Unauthorized access requested for stream: {stream}");
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(&stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(&stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(&stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: self.start_time.clone(),
+                end_time: self.end_time.clone(),
+                num_bins: 1,
+            }
+            .get_bin_density()
+            .await?;
+            let counts = CountsResponse {
+                fields: vec!["start_time".into(), "end_time".into(), "count".into()],
+                records,
+            };
+
+            // Retrieve distinct values for source identifiers
+            // Returns None if fields aren't present or if query fails
+            let ips = self.get_distinct_entries(&stream, "p_src_ip").await.ok();
+            let user_agents = self
+                .get_distinct_entries(&stream, "p_user_agent")
+                .await
+                .ok();
+
+            responses.push(PrismDatasetResponse {
+                info,
+                stats,
+                retention,
+                hottier,
+                counts,
+                distinct_sources: json!({
+                    "ips": ips,
+                    "user_agents": user_agents
+                }),
+            })
+        }
+
+        Ok(responses)
+    }
+
+    /// Retrieves distinct values for a specific field in a stream.
+    ///
+    /// # Parameters
+    /// - `stream_name`: Name of the stream to query
+    /// - `field`: Field name to get distinct values for
+    ///
+    /// # Returns
+    /// - `Ok(Vec<String>)`: List of distinct values found for the field
+    /// - `Err(QueryError)`: If the query fails or field doesn't exist
+    async fn get_distinct_entries(
+        &self,
+        stream_name: &str,
+        field: &str,
+    ) -> Result<Vec<String>, QueryError> {
+        let query = Query {
+            query: format!("SELECT DISTINCT({field}) FOR {stream_name}"),
path: src/prism/logstream/mod.rs
user: login: de-sh
body: You can ignore all SQL injection issues in the codebase as the SQL queries are run against immutable data streams
created_at: 2025-03-13T11:39:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query and the time range for the query.
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+    /// ISO 8601 formatted start time or human-readable time expression
+    start_time: String,
+    /// ISO 8601 formatted end time or human-readable time expression
+    end_time: String,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    /// successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        if self.streams.is_empty() {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                warn!("Unauthorized access requested for stream: {stream}");
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(&stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(&stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(&stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: self.start_time.clone(),
+                end_time: self.end_time.clone(),
+                num_bins: 1,
+            }
+            .get_bin_density()
+            .await?;
+            let counts = CountsResponse {
+                fields: vec!["start_time".into(), "end_time".into(), "count".into()],
+                records,
+            };
+
+            // Retrieve distinct values for source identifiers
+            // Returns None if fields aren't present or if query fails
+            let ips = self.get_distinct_entries(&stream, "p_src_ip").await.ok();
+            let user_agents = self
+                .get_distinct_entries(&stream, "p_user_agent")
+                .await
+                .ok();
+
+            responses.push(PrismDatasetResponse {
+                info,
+                stats,
+                retention,
+                hottier,
+                counts,
+                distinct_sources: json!({
+                    "ips": ips,
+                    "user_agents": user_agents
+                }),
+            })
+        }
+
+        Ok(responses)
+    }
+
+    /// Retrieves distinct values for a specific field in a stream.
+    ///
+    /// # Parameters
+    /// - `stream_name`: Name of the stream to query
+    /// - `field`: Field name to get distinct values for
+    ///
+    /// # Returns
+    /// - `Ok(Vec<String>)`: List of distinct values found for the field
+    /// - `Err(QueryError)`: If the query fails or field doesn't exist
+    async fn get_distinct_entries(
+        &self,
+        stream_name: &str,
+        field: &str,
+    ) -> Result<Vec<String>, QueryError> {
+        let query = Query {
+            query: format!("SELECT DISTINCT({field}) FOR {stream_name}"),
path: src/prism/logstream/mod.rs
user: login: coderabbitai[bot]
body: @de-sh I understand and will note this architecture detail for future reviews. Thank you for clarifying that SQL injection concerns are mitigated by running queries against immutable data streams.

---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: de-sh
PR: parseablehq/parseable#1236
File: src/prism/logstream/mod.rs:332-332
Timestamp: 2025-03-13T11:39:52.394Z
Learning: SQL injection concerns can be ignored in this codebase as all SQL queries are run against immutable data streams, limiting the potential impact of any injection.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-03-13T11:39:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Name of the stream
+    stream: String,
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query
+#[derive(Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    ///    successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        let is_empty = self.streams.is_empty();
+        if is_empty {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                // Don't warn if listed from Parseable
+                if !is_empty {
+                    warn!("Unauthorized access requested for stream: {stream}");
+                }
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: "1h".to_owned(),
+                end_time: "now".to_owned(),
+                num_bins: 1,
path: src/prism/logstream/mod.rs
user: login: de-sh
body: ```suggestion
                num_bins: 10,
```
created_at: 2025-03-17T09:06:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Name of the stream
+    stream: String,
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query
+#[derive(Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    ///    successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        let is_empty = self.streams.is_empty();
+        if is_empty {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                // Don't warn if listed from Parseable
+                if !is_empty {
+                    warn!("Unauthorized access requested for stream: {stream}");
+                }
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: "1h".to_owned(),
+                end_time: "now".to_owned(),
+                num_bins: 1,
path: src/prism/logstream/mod.rs
user: login: de-sh
body: @praveen5959 is this enough?
created_at: 2025-03-17T09:07:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -185,6 +194,168 @@ async fn get_stream_info_helper(stream_name: &str) -> Result<StreamInfo, StreamE
     Ok(stream_info)
 }
 
+/// Response structure for Prism dataset queries.
+/// Contains information about a stream, its statistics, retention policy,
+/// and query results.
+#[derive(Serialize)]
+pub struct PrismDatasetResponse {
+    /// Name of the stream
+    stream: String,
+    /// Basic information about the stream
+    info: StreamInfo,
+    /// Statistics for the queried timeframe
+    stats: QueriedStats,
+    /// Retention policy details
+    retention: Retention,
+    /// Hot tier information if available
+    hottier: Option<StreamHotTier>,
+    /// Count of records in the specified time range
+    counts: CountsResponse,
+    /// Collection of distinct values for source identifiers
+    distinct_sources: Value,
+}
+
+/// Request parameters for retrieving Prism dataset information.
+/// Defines which streams to query
+#[derive(Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct PrismDatasetRequest {
+    /// List of stream names to query
+    #[serde(default)]
+    streams: Vec<String>,
+}
+
+impl PrismDatasetRequest {
+    /// Retrieves dataset information for all specified streams.
+    ///
+    /// Processes each stream in the request and compiles their information.
+    /// Streams that don't exist or can't be accessed are skipped.
+    ///
+    /// # Returns
+    /// - `Ok(Vec<PrismDatasetResponse>)`: List of responses for successfully processed streams
+    /// - `Err(PrismLogstreamError)`: If a critical error occurs during processing
+    ///
+    /// # Note
+    /// 1. This method won't fail if individual streams fail - it will only include
+    ///    successfully processed streams in the result.
+    /// 2. On receiving an empty stream list, we return for all streams the user is able to query for
+    pub async fn get_datasets(
+        mut self,
+        key: SessionKey,
+    ) -> Result<Vec<PrismDatasetResponse>, PrismLogstreamError> {
+        let is_empty = self.streams.is_empty();
+        if is_empty {
+            self.streams = PARSEABLE.streams.list();
+        }
+
+        let mut responses = vec![];
+        for stream in self.streams.iter() {
+            if Users.authorize(key.clone(), Action::ListStream, Some(stream), None)
+                != crate::rbac::Response::Authorized
+            {
+                // Don't warn if listed from Parseable
+                if !is_empty {
+                    warn!("Unauthorized access requested for stream: {stream}");
+                }
+                continue;
+            }
+
+            if PARSEABLE.check_or_load_stream(stream).await {
+                debug!("Stream not found: {stream}");
+                continue;
+            }
+
+            let PrismLogstreamInfo {
+                info,
+                stats,
+                retention,
+                ..
+            } = get_prism_logstream_info(stream).await?;
+
+            let hottier = match HotTierManager::global() {
+                Some(hot_tier_manager) => {
+                    let stats = hot_tier_manager.get_hot_tier(stream).await?;
+                    Some(stats)
+                }
+                _ => None,
+            };
+            let records = CountsRequest {
+                stream: stream.clone(),
+                start_time: "1h".to_owned(),
+                end_time: "now".to_owned(),
+                num_bins: 1,
path: src/prism/logstream/mod.rs
user: login: praveen5959
body: @de-sh  Yeah this will work.
created_at: 2025-03-17T10:15:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: coderabbitai[bot]
created_at: 2025-03-13T11:18:34Z
body: <!-- This is an auto-generated comment: summarize by coderabbit.ai -->
<!-- walkthrough_start -->

## Walkthrough
This change adds a dataset service endpoint to the HTTP server. The implementation updates the route configuration in both the query server and server modules by registering a new method for datasets. A new asynchronous handler is introduced to process POST requests for datasets, including JSON parsing and authorization checks. Additionally, new request and response structures, along with an updated error enum in the Prism module, support dataset queries. Finally, minor improvements have been made to JSON conversion for Arrow record batches.

## Changes

| File(s) | Change Summary |
|---------|----------------|
| `src/handlers/http/modal/query_server.rs`, `src/handlers/http/modal/server.rs`, `src/handlers/http/prism_logstream.rs` | Added new dataset endpoints: registered `.service(Server::get_prism_datasets())` in query server, defined a dataset route (`/datasets`) with authorization in server, and implemented the async `post_datasets` handler. |
| `src/prism/logstream/mod.rs`, `src/response.rs` | Introduced `PrismDatasetRequest` and `PrismDatasetResponse` structs, implemented the `get_datasets` method with extended error variants, and simplified JSON conversion in the `to_http` method. |
| `src/utils/arrow/mod.rs` | Updated the `record_batches_to_json` function to accept owned `RecordBatch` instances and modified the writing loop and corresponding test case. |

## Sequence Diagram(s)
```mermaid
sequenceDiagram
    participant Client
    participant Server
    participant Handler
    participant PrismModule

    Client->>Server: POST /datasets (JSON payload)
    Server->>Handler: Forward dataset request
    Handler->>PrismModule: Execute get_datasets()
    PrismModule-->>Handler: Return dataset info or error
    Handler->>Server: Wrap response as JSON
    Server-->>Client: HTTP JSON response
```

## Possibly related PRs
- **parseablehq/parseable#1238**: The changes in the main PR are related to the addition of the `get_prism_datasets` method in the `Server`, which is also introduced in the retrieved PR as part of the `DiskWriter` functionality, indicating a connection in the context of handling dataset requests.
- **parseablehq/parseable#1226**: The changes in the main PR are related to the addition of the `get_prism_datasets` method in the `Server` implementation, which is directly connected to the new `post_datasets` function introduced in the retrieved PR, as both involve handling dataset-related requests.
- **parseablehq/parseable#1185**: The changes in the main PR are related to the addition of the `get_prism_datasets` method in the `Server` struct, which is also reflected in the retrieved PR where the `get_prism_datasets` method is introduced in the same `Server` implementation.

## Suggested labels
`for next release`

## Suggested reviewers
- nikhilsinhaparseable

## Poem
> Hop, hop, along the code trail we go, 🐇  
> New endpoints sprout like carrots in a row.  
> Datasets and logs now dance in sync,  
> Each function and method in perfect link.  
> With gentle hops, our code grows neat—  
> A bunny’s delight in every beat!  
> Happy coding in the digital heat!

<!-- walkthrough_end -->

<!-- announcements_start -->

> [!TIP]
> <details>
> <summary>⚡🧪 Multi-step agentic review comment chat (experimental)</summary>
> 
> - We're introducing multi-step agentic chat in review comments. This experimental feature enhances review discussions with the CodeRabbit agentic chat by enabling advanced interactions, including the ability to create pull requests directly from comments.
> 						- To enable this feature, set early_access to true under in the settings.
> 
> </details>

<!-- announcements_end -->

---

<details>
<summary>📜 Recent review details</summary>

**Configuration used: CodeRabbit UI**
**Review profile: CHILL**
**Plan: Pro**

<details>
<summary>📥 Commits</summary>

Reviewing files that changed from the base of the PR and between 6bf49c04a0a05b61ad65d40d61655a94d70ef045 and ab16243d2917f8d1fee03def79a30d4ae650e148.

</details>

<details>
<summary>📒 Files selected for processing (1)</summary>

* `src/prism/logstream/mod.rs` (5 hunks)

</details>

<details>
<summary>🚧 Files skipped from review as they are similar to previous changes (1)</summary>

* src/prism/logstream/mod.rs

</details>

<details>
<summary>⏰ Context from checks skipped due to timeout of 90000ms (10)</summary>

* GitHub Check: Quest Smoke and Load Tests for Standalone deployments
* GitHub Check: Build Default aarch64-apple-darwin
* GitHub Check: coverage
* GitHub Check: Build Default x86_64-unknown-linux-gnu
* GitHub Check: Build Default aarch64-unknown-linux-gnu
* GitHub Check: Quest Smoke and Load Tests for Distributed deployments
* GitHub Check: Build Kafka x86_64-unknown-linux-gnu
* GitHub Check: Build Default x86_64-apple-darwin
* GitHub Check: Build Kafka aarch64-apple-darwin
* GitHub Check: Build Default x86_64-pc-windows-msvc

</details>

</details>
<!-- internal state start -->


<!-- DwQgtGAEAqAWCWBnSTIEMB26CuAXA9mAOYCmGJATmriQCaQDG+Ats2bgFyQAOFk+AIwBWJBrngA3EsgEBPRvlqU0AgfFwA6NPEgQAfACgjoCEYDEZyAAUASpADK2VmgqyjcEj2wAbb5AokAI7Y0riQ4rjedJAARABmJNRcvEjMPPiIYbTUaIgkuMgAglYAkjGQAO65kIjYAszqNPRy4bCe2Hl8SmCIsOgY9KjwShjicfDRLbhtkBhOApSQAIwATADMAGwaMDPc2BTcGZ74ca2otijIBCijFIrYDJ5osyQVkMUlkGS0h/CjkHF8HxDpk/kRINlcLl8sgAt5qNFrtNPFYKKkAYlcPtPH9WiiXHkVFF0NxuN54AxqPB8BgNO4ZkpEAw0dxxDSbgxvNhGf0vgAPNDMMnHU7PALBUL9egBRCHDB5AFA5jURV8ZHvUrbDz+IIhTKXGq4CgPLEBQZYABS9gA8gA5VXK3AAGhq3FE8DisjB6ENAUFKBGYwmFC1MxIAqFxN4+Akw2ivXwFWQYt1ksBapmmT9aRiFChAGtBXEiPnuOVMPR8HhyeQrmG+W6xNEZXK8i6KggGH0/pzudIIfltFFzenHdSsCoq2F1ZDoc6ag8+tVmZjx+F4GwXVnEmlcLI3S7vPhwYgqxRHi6K5AJC5qR1DVTQQxYSR4U1wvgbqRQeyr5kgWgpDbCUYTRrGPJMNgozICcOpMBQtCIC6PZcrQ3qZC4YRXt865sEhUp4h+UJ+JB/ywQE8GITwdyPIgeT0BU6gIFg6qyu64yIhunhxFQbDbIUtBoWyGBoL4sjIWEU41v2rFGia2KqjqNCjGuV5oaCGBiDUZ60Ze3jTFWRB9MiCocd4VEuJ4CYVBOyBzL4NyEeGgrCnSBjahcomnhyqH9s8XaiPm5L6n8aGUuIGDgtMKrqmgeCwECkCwNUNCZIiMxHuCYKhKpAyQBKrgAlBYjjqJ6jieggnREwrDsMg6aMPCaJ7peeVgXG9C0PgDBOOwVLsg1mDyOQbyJcwigehM9AJNQ2KIKGKJ2NozBXJ+ZDJZpnjqpS3AqPA5LiP2sHqlYBKJAIUZvqOkAtGQRLeh0lCrTcNBUFpjHTBCOR5AUkDjQEXxxAkJVSN4shuQAYolcT7MiXSDvt+EnXYlJYAs6AMLR9HoGEADa0yoDW+YALoABSwLguDcIgHAAPS00QTF1BoNW07tFCEhdJCwIEbNnUSJBsz43i06smwAJRuRYkAAMIsGw0EOE4yquEYJQsTMNUK79eQleyx0zLYl6QI9fAiWw9C8GgUhkAArAAnA76R/O+U6tDFMwAAbboKAD65skJ7BoNHR3o8SwhE+2kUSg18tzyLi6ofDqso0gqWbyQEoaoFr7BJfARnkkZv3PIcyniKJ/BSBzBcU456obsK2v9VgBueCNGqfN8vyjFu2BEN+EVRcl04zChHRrm3kDe0aO7+4KgcGuQWMuPIDWcree4ETVwrKdIyCJwyORKfs5DNPISeakYtqfmggnqKVJHy3V/BdEgPWh2nlSUJ4FFAko5pCK50VrrYSyEBgUipJFd2o9UBVDrJ4FIKs14kDoAINADB8zURjB1HURAXBoWgcjBaChar/E6FIZMyYFDyjkiDTwzIH4SnQFxegKpG53FjEQmYHwADk9VirCTKlvbC8p9jemimEdQf1UG/VNvyRsUJhLJjyuKbA8AAja3mkYfQxhwBQBwrBOKBBiBkGUO+YBnBqL8GEKIcQlCbryCYEoKgqh1BaB0LokwUA4BDBUTgExpByBUAsc/UYXAqBvFqM4QqLRnHKDcZobQugwCGD0aYAwiBzy0w2rQKIHMcmU24LTca2QRYFVkL7ChlANAcw4AYGIjSDDS0KCUUxwSET0Gicg/gpwuyYG/PSTwaEgYvTuLQB4fkXhRMoLGR4ccfj4Bdh+QinsACKIRXD2FmZQIOjcojN2Er01Zp0ObnSiNsig1cg5Gm0JoBwjZJqUjEhJSqVF1TSWnhoChFISCk0udXDgHBSC4F9ikRAzBfazh+ogUm4txY3M/NtGk4wiD7Bbscu4eB+xQRcas8FkL0F5DBdQWAcKg5MnwG6bOyZ75HMBL4RMCD+RICHjUWZvzFKexBWCtEELfaZSjuS8BYUoHDw9lZHZfBGIOQwImJKFZiRqNCC+N8iJPzQphI5O+Qk1zXEyr6Hc81IA30gNJZAFQf46nGlISsfBSmTWiInSgK1jnhlZWHQRj9yoAG4gEbVINpKIYMbgSHwN4BxsU6WT1FNM9lVzOXdyWaMNy5hLCFH0uY8cz0ZyiCai3GCpxwyHDzNERKewLoUjjhECYiAdGQAALL5ASmwqqtAuCeziFgHlBKoXfRhOSxynsAW7KHVkhgOTFVPUKVTEpihRK0wqVUqVtTECewaU0gwEAwBGHHZOgY+TEAzuKaUhd1SQx1I3TEZp6a2lBPMfGZWq9jn9MitIIZjAA39hduMyZPoO4/PmUm5Zh9PDDqlXsyMJBDnjm2PYR54xnlg2Nh3NgBl6DcvyLy1IfaoQwr2bSgB7ZOx9BXAiahlqBA1CYG6RSuSazgk1bgMAcJOk6glJkY1vjkBoZbQOcYtYfRWGtPYaA/gpxPFHog0l09aZMbXcRik3ZkBE3VasimVMgW9sFbPQUWmMigvk0HGGmlhL8WjTSUSyGUBSM0r5ZM8UgTwAAF4YoCpg+qiVYp63lNPAA4vkewunmDq0BJ7F0nsAu4CC9QBTBFIv5BsPkdg45PYkPdRpE8HL5lMAwKi9FRzQPT1y/lgIvssWpSDrxxQBpsDcEhGp8eSgzh2VeF9PD+R41zKeEeSKiA4yEV4CQWMVZEAhqUAJ6IOnsxdd+SQmMygHIZzEApKeFXvSaO0L5l2GD5wfWMjMAD2XPALG9Dq6I+23aaJwdAoaLLMtFVM96vcqab3vEzSE7NKzc0b0+1/WCxagSu2BHUckDBq0P3fVuxtzaavnfbdPCtAJu1Yd7fJuFug9AOBo4vXEy2wjgauaO3E3tsn0enRpk986RbntXeuxp16t0ZL3eTgplO2Z8shdNncq76kM7e609pD6ulPsKrBV9gz3IMg9KcH99xaL/ra7kWQmlYB3DlfeEzPnEcGdw3OOLHYlPYKxv2AAEtAaAVh2N6l+tcAIRoJhSHa/rmlj3tcYMeKyfxVKhF+CtHaHgaBZBHjvscvctHPaolSAAEX7bgJLHHcBByvJgaepuikJ5t2lyAIF+S3LENQvIn8sD5hIPIOJVnvSe3DPn0FRf+s0l9qXyp4dIXilJgAMnFOLAA/OFyoJGDRoCqOoaIDVKB3D4PRsEJCtdHORBOSm0GvcrPt2iYbW0j4dbCBPbhYGeXo/r+OJvZeEUyPQy6JDZaNbSbOfQf39pdrB/wHfF0E24r6QkbfLAy+t5MZQKcHKoHs/qHqgO1AAiQjKD4Gyv/nAlQKSE6hONPJRkChaKeBgEHIICIFpAsOmCdiQBtvkKfHQOZrqpZi8oRI3EDg+DQFogqk7gsGQCbPVmxtcE1mBmgTSP3p7OnlTJnqEP3leJHpzrHtvvwZkP3obl2OgADMvPvM+g1OqB3HPs9uDDsJ4BlmyphqCn8GFm7kcptn8MgMVF+rQFLOmh9gWt9gyHmi4FYQDg2EDmWiDpWuDilodHWtDk2uhm8nQB2kjsrppMjukJkHrjCqTExuVkEFwNaKyOOMAJwRgMAFHhCqIXOOIbgHoHoC6OKFwLwdwBkeLJjpAElrUPpMAPsiUdIHKC4i6CkcwAADLHhRwACiFAk+egeyWApOE6rOR67O2mzRwWdO0OAAareALOEPuE8G2h2okV0dPCzlOmzkUhzjhtzoKHTopOqJ7CCIZnHmuufi2nSPzkzjupktkgSrTBscwHOmYZevzi0nemYiEo+jEvIOLl+p4dqBLt+rcPLu3G1rOL6JnH5HlNVlRA1NPtAv/gVLWvXF7PUVVvcFEPxHGi2GnFZHQqaCQBFvUWkT9KUa2IHMhIRnQC6GQDtGUWKgOFCIjOgAIG7M8FHOAr2IQllDrEoh6s+DkclipOyIcGDhVAlNOMGDcKOC3JflWIrORKIP/PhGpB6qZtpPsPMnGCpBxBzKQQ/OQdZs8B3MqvqPjtiHiSIXHhkf3kMP8RMo8PQNcBNn8LfgvK9J5nwBUt6PJqyahGdmaqysclHLMAvNolLvvlhkZkcTVkMFBtrBdkxCTvieaamBIYptIdGCbsgIkNIWxAwI6l0sFryQ7hvqqsNpgLZhKWAtvCwNwPtP2BifKH8dcM8FIGIIlLBMITHomanPWdntxhGeaGyf2BPolNCeCA1FBMYglGiM5tEB7vvARIgPmPANTIaoKHWCqKjHKmEOjLOXRCQTnmEGvo7v2OpBFFpOwOvq6a6OxFNACBMOZAfDfiuStC6COekOXPAJXBUgCEOHNJAEQG9CQDDGJJDLDG0BQP9LiScpzk0UQK0e0UCEHGQE4AaMWhWI1nZn2HGkOXwDeGiKWZea+dhaqqwZ+CKeuIsMqCJKQNrC6F+eGKIHgOOC6OIGwIHjXJFK1Gwo5lOS3CQjPLNIgL7PElVrDpWKcATBmXMGkKgHVg1naZ+PbqfCSNGCkAiJAObpbjQfePEpeSZICW8NhVMW6MGdaNXFZsxZrF8XHBtPMuqCoeQeVGHjMKUj4CdvIDqt6DvAEG0PKJIL/EmVhKotUZiQqgemHIlLCZsrWnpDSOCPtigEKJwtEIZZRYBNBuwOYe9q9FYUiDYb9vYUWo4aWnal4K4RDh4fWraG1vjr4QjrsXUCCVpAmWIcFd2WOpcZztcUMdmHcSMVAFVVEtibVf4Q1TVc1ekQFQsT0WsRCl1bBcFr1RzOulAN4XxvDiNVRoEeDl2n+WGQcaTHkN4HEC6M3lwNssXgANKn7FGlHQHACjGiDJFmktVdl5DZHWDQXdU7htEdFTWVHjWEmTXtUTpXE3GLVrpeGiXDXTxbXBEH5KliC+znm1qd6HXHVPnzxsBcAd5ZguhmQI441GhFGpJVFlG4D3WPVBZoiRTvUbKUCyA/VAidFDr/XPUTWJ5TV7qg1fWCjg3LWQAtFSXXgTH/DrVp74CUzBikxmC4yt7EzqUS3QDBiM0UBn4k5I5IVpD1EwVwWT6c0dWpBzVRx83Q6C3IW4Ufmi2zHTx02uDS2y13DMDy220M3wWq1TUa1C3a083MAq360g2dVg2lJ9UC1C0W2lnQ2exK1sCnJ5D21y0wBcSx0kAq1q3dGe3IXe3zXZh+3A0zW3FB2KAh1m1pDh1W0AIdotF8gMU0Dx2O3y1V010p1u1p2I4NWa0fWpA63Ba50k5c2B0+0m1QAl3C14Xl1+HTyFDxR10sDy27bwB8i+woEcCp0e3t1e2fXZ3fVu3+351G0LXB1LVXo6LM7ZJ1l5C87H1PFC6vEi7vEvpfEfq/E8Z3w4iPmewEC+yU4iU+FTzrKbKyBEmYkUpDV/BhoRpPA1D7JPIYqwQJRvA7FCDoFRGUSHHh1cwGiHB7BqpmHqGvwFx/CVz7LpWjBuZ+jvj6ltbNkEB8Cex/wISHGSIKC+B2LNiAU/ybQ5pexJaUQABC1AXYXRGEnDAIjtixr4cQtScpDDPZCAD5r0FsH5NAhoJA3ACqMgqCWA12tqnFqySDje9DiEQc6DxIqAcqbwaEFEkQ7locRAZ8hEUB+kTlYGhjvs6CuAAUgln9+jGB+ha4V+DETE4jR1UjqDsjOcX6Xw5IDQIkqUhEUEchdEz6ZGRyf9hjaDdiQIW4wWNYEimsignghuxIyoO2fweTVkC8fjDlL2H6hlr5hhvmJhAy5J6AzKdD0jtAbjAjbQXj+AvsPjxmXq7IqAoIDkxarD8lJ8FASBnst1+kNy0xKZymoVeSiBkclTlFwSrTN0gFQIJAmVGa2Vyi1hwythf2vmDhJawOpVYO5VtalVt8lhxzuVpz+VxzlzThJVFatz7h8JlqAMyozWh8qAIykjx9Zxu62SjF3gR6LgdwFQ4NfOTS1996t9C499nxzT3xllWLCJDCBToa4aTuDqiGOVSKXsrj7jnjvs3j6BgzT2v4eU6gxhiAaVeL4QkolICons7m+YSNQoe4/TyDn9lL3T0gsj3EQzWA/Wdjs0AM8C6AH2ams5rIPoY2nKbZPD/8/DHjsAQcAQCQAQIjpMnsHeuMHeWrCEOrXYxMnsRRRhNAoeRiNQYOIo08lrtA1rerwtXI/YJrZrHrXrtrksOwETzTUDkYjqzKr5splEN0YrMg7ljKjEMJGidi/ANkT0CAajmrHTXrQjUIIjISYF7sWjuoGihBhrlJMk5L0Gbk6slB0ZfUqTpw7TlEXTur0gNLfTAzFlngEJxyFQzUhBcbVLPT6jOzTBvx00Yju+4InsQ7o+IYi7NAHbnjpM6TveiKPoR4VKMCNmD6ME1cXwGCfQhj8bur28VmyAC7zUNSK7fyhjW7ruJLkCRyVmTKikEFf5VA9kLgNCRo4aVciw6oK73oaZ+8LoZcKWVm7lmM8s86MBMuHDYQr54+p7cE/8bkAkZBIkFBRWqUYQ9l3RvL/LrIlSPj3bKD/8a7PT/e205DMahE1DrZpw57Y7/YyUGjTBslbGre08UQB5VGAAvNeKICAOaxQMTD6tu57IJ/4CJ2JwwBJ1JzJ7yXEFECVHvp+uG0VsR1A7KziSsiq5JJmxhoG90wW5gLRG5KZYtt4H2wqM/dZdZ5vpKwyxgPwgCFEHyPAGoAdO5XlP1pG56IRIR4wNCI4tajdsPICWPolAaxwzZ0YGmllVml/C8wOG819h88Va/Dc1Wr81DitVDTK7EwpLx00BtcEaK52700KzSBux0zTJAGaxa3m908TMTVjnM+TQ9QwMAA2mgNwMAFTWCC6OMb61kczX3VC+IDC7THC4mHzZAIAEmEbdVGO1tX1LtLjX6T2NuMFnurXXN10gd1/Xg3w3o3DuHFkAk3IQ03u90LsL8FCLh9ENpxuiBg3iCyxyxihAqLbGliESw+6LPScSBTriagSSniqS33+ipCDQOhiEURsYrwdAVSUIeYKSaSP3tsmwAA7AABwrAAAMawRP9sAALBsAIBsKIEsPbCsET2sCQBTwwETyQEsLQAwLQPbNkET7QLT5zzj/DxAJAFTysAT2sEsBsEoLbHfEsAIKT3EATysEsLbNLysPj1TwIAkJr9z7bAIDz7QKT6T/bKLz9/bKT7QHEA7CsPz2sGgDT6T1T2sFT2gIL7QFTyr1LxsG71zwT0sKTysGgBsNz0sJbwj/bBsET6TwILbCsHEAwKT0sATwTx7yQMrzzysLnwIAT8DOgm73EEr/bGsHEGsCsFH+L2sLbLbOHyQPbNb2gKz1T/jxgvj2sAT5jLbKT570Tz70sEsAwEsC3xsGsLQJH14gj0L7r0T571T3EOT3EBsHELQET8H48DL6sAP/nysLQCsB7+TwT7bILxb9P+L7T3EFT/bCn0f2gKT0b+H3fBsLbN7zb+H6/wr9T7QAT6T4Ba71tjV8oAKgWXofwn4O80+cQQXksASAkByeE2AnvbBb429M+r/f/ksCp5E9ReYvKAFrHUC+xhgglAIGj0tSdMcI+gIAA== -->

<!-- internal state end -->
<!-- tips_start -->

---



<details>
<summary>🪧 Tips</summary>

### Chat

There are 3 ways to chat with [CodeRabbit](https://coderabbit.ai?utm_source=oss&utm_medium=github&utm_campaign=parseablehq/parseable&utm_content=1236):

- Review comments: Directly reply to a review comment made by CodeRabbit. Example:
  - `I pushed a fix in commit <commit_id>, please review it.`
  - `Generate unit testing code for this file.`
  - `Open a follow-up GitHub issue for this discussion.`
- Files and specific lines of code (under the "Files changed" tab): Tag `@coderabbitai` in a new review comment at the desired location with your query. Examples:
  - `@coderabbitai generate unit testing code for this file.`
  -	`@coderabbitai modularize this function.`
- PR comments: Tag `@coderabbitai` in a new PR comment to ask questions about the PR branch. For the best results, please provide a very specific query, as very limited context is provided in this mode. Examples:
  - `@coderabbitai gather interesting stats about this repository and render them as a table. Additionally, render a pie chart showing the language distribution in the codebase.`
  - `@coderabbitai read src/utils.ts and generate unit testing code.`
  - `@coderabbitai read the files in the src/scheduler package and generate a class diagram using mermaid and a README in the markdown format.`
  - `@coderabbitai help me debug CodeRabbit configuration file.`

Note: Be mindful of the bot's finite context window. It's strongly recommended to break down tasks such as reading entire modules into smaller chunks. For a focused discussion, use review comments to chat about specific files and their changes, instead of using the PR comments.

### CodeRabbit Commands (Invoked using PR comments)

- `@coderabbitai pause` to pause the reviews on a PR.
- `@coderabbitai resume` to resume the paused reviews.
- `@coderabbitai review` to trigger an incremental review. This is useful when automatic reviews are disabled for the repository.
- `@coderabbitai full review` to do a full review from scratch and review all the files again.
- `@coderabbitai summary` to regenerate the summary of the PR.
- `@coderabbitai generate docstrings` to [generate docstrings](https://docs.coderabbit.ai/finishing-touches/docstrings) for this PR.
- `@coderabbitai resolve` resolve all the CodeRabbit review comments.
- `@coderabbitai configuration` to show the current CodeRabbit configuration for the repository.
- `@coderabbitai help` to get help.

### Other keywords and placeholders

- Add `@coderabbitai ignore` anywhere in the PR description to prevent this PR from being reviewed.
- Add `@coderabbitai summary` to generate the high-level summary at a specific location in the PR description.
- Add `@coderabbitai` anywhere in the PR title to generate the title automatically.

### CodeRabbit Configuration File (`.coderabbit.yaml`)

- You can programmatically configure CodeRabbit by adding a `.coderabbit.yaml` file to the root of your repository.
- Please see the [configuration documentation](https://docs.coderabbit.ai/guides/configure-coderabbit) for more information.
- If your editor has YAML language server enabled, you can add the path at the top of this file to enable auto-completion and validation: `# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json`

### Documentation and Community

- Visit our [Documentation](https://docs.coderabbit.ai) for detailed information on how to use CodeRabbit.
- Join our [Discord Community](http://discord.gg/coderabbit) to get help, request features, and share feedback.
- Follow us on [X/Twitter](https://twitter.com/coderabbitai) for updates and announcements.

</details>

<!-- tips_end -->
author_association: CONTRIBUTOR
user: login: praveen5959
created_at: 2025-03-14T07:16:49Z
body: `stream_name` is missing from the stream level entry. 
author_association: CONTRIBUTOR