Repository: codice/ddf
Pull request #6870: [2.29.x] Query metrics updates
Author: login: zkirksey
Created: 2025-03-11T01:19:37Z | Merged: 2025-03-20T22:45:33Z

#### What does this PR do?
Adds new query metric for start date of query and fixes bug with federated sources

#### Who is reviewing it? 
<!--(please choose AT LEAST two reviewers that need to approve the PR before it can get merged)-->

#### Select relevant component teams: 
<!--
@codice/build 
@codice/continuous-integration 
@codice/core-apis 
@codice/data 
@codice/docs 
@codice/io 
@codice/ogc 
@codice/security 
@codice/solr 
@codice/test 
@codice/ui 
@codice/website 
-->

#### Ask 2 committers to review/merge the PR and tag them here.
<!--
If you don't know who to ask, you can request reviews in https://groups.google.com/forum/#!forum/ddf-developers .
(please choose ONLY two committers from below)
@ahoffer
@andrewkfiedler
@andrewzimmer
@AzGoalie
@bdthomson
@blen-desta
@brendan-hofmann
@brianfelix
@cassandrabailey293
@clockard
@coyotesqrl
@emmberk
@figliold
@garrettfreibott
@glenhein 
@gordocanchola 
@hayleynorton
@jlcsmith
@josephthweatt
@jrnorth
@lambeaux
@lamhuy
@leo-sakh
@mcalcote
@millerw8
@mojogitoverhere
@oconnormi
@paouelle
@pklinef
@ricklarsen - Documentation
@ryeats
@rymach
@rzwiefel
@shaundmorris
@smithjosh
@stustison
@vinamartin
@zta6
-->

#### How should this be tested?
<!--(List steps with links to updated documentation)-->

#### Any background context you want to provide?

#### What are the relevant tickets?
Fixes: #____

#### Screenshots
<!--(if appropriate)-->

#### Checklist:
- [ ] Documentation Updated
- [ ] Update / Add Threat Dragon models
- [ ] Update / Add Unit Tests
- [ ] Update / Add Integration Tests

#### Notes on Review Process
Please see [Notes on Review Process](https://codice.atlassian.net/wiki/spaces/DDF/pages/71946981/Pull+Request+Guidelines) for further guidance on requirements for merging and abbreviated reviews. 

#### Review Comment Legend:
- ✏️ (Pencil) This comment is a nitpick or style suggestion, no action required for approval. This comment should provide a suggestion either as an in line code snippet or a gist. 
- ❓ (Question Mark) This comment is to gain a clearer understanding of design or code choices, clarification is required but action may not be necessary for approval.
- ❗ (Exclamation Mark) This comment is critical and requires clarification or action before approval.



Diff:
diff --git a/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/impl/operations/QueryOperations.java b/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/impl/operations/QueryOperations.java
index bdfb05093dc7..32cf9394c9b2 100644
--- a/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/impl/operations/QueryOperations.java
+++ b/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/impl/operations/QueryOperations.java
@@ -15,8 +15,6 @@
 
 import static ddf.catalog.Constants.QUERY_LOGGER_NAME;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
 import ddf.catalog.Constants;
 import ddf.catalog.core.versioning.DeletedMetacard;
 import ddf.catalog.core.versioning.MetacardVersion;
@@ -209,8 +207,6 @@ QueryResponse query(
       boolean overrideFanoutRename,
       boolean fanoutEnabled)
       throws UnsupportedQueryException, FederationException {
-    long queryStart = System.nanoTime();
-
     FederationStrategy fedStrategy = strategy;
     QueryResponse queryResponse;
 
@@ -257,7 +253,6 @@ QueryResponse query(
       queryResponse = processPostQueryAccessPlugins(queryResponse);
       queryResponse = processPostQueryPlugins(queryResponse);
       putMetricsDuration(queryResponse, QM_POST_QUERY, System.nanoTime() - postQueryStart);
-      putMetricsDuration(queryResponse, QM_TOTAL_ELAPSED, System.nanoTime() - queryStart);
 
       log(queryResponse);
 
@@ -289,35 +284,6 @@ private void log(QueryResponse queryResponse) {
         }
       }
     }
-    Map<String, Serializable> requestProperties =
-        queryResponse.getRequest() == null ? null : queryResponse.getRequest().getProperties();
-    Map<String, Serializable> allProperties =
-        collectQueryProperties(requestProperties, queryResponse.getProperties());
-    if (Boolean.TRUE.equals(allProperties.get("metrics-enabled"))) {
-      QUERY_LOGGER.info("QueryMetrics: {}", getQueryMetricsLog(allProperties));
-    }
-  }
-
-  protected static String getQueryMetricsLog(Map<String, Serializable> properties) {
-    Map<String, Serializable> metricsMap = new HashMap<>();
-    // Combine the request and response metrics to log
-    metricsMap.put("total-duration", properties.get(QM_TOTAL_ELAPSED));
-    metricsMap.put("prequery-duration", properties.get(QM_PRE_QUERY));
-    metricsMap.put("query-duration", properties.get(QM_DO_QUERY));
-    metricsMap.put("postquery-duration", properties.get(QM_POST_QUERY));
-    HashMap<String, Serializable> sourceDurationMap = new HashMap<>();
-    properties.entrySet().stream()
-        .filter(e -> e.getKey() != null && e.getKey().startsWith(QM_TIMED_SOURCE))
-        .forEach(e -> sourceDurationMap.put(e.getKey().split(QM_TIMED_SOURCE)[1], e.getValue()));
-    metricsMap.put("source-duration", sourceDurationMap);
-    HashMap<String, Serializable> additionalQueryMetrics =
-        (HashMap<String, Serializable>) properties.get("additional-query-metrics");
-    if (additionalQueryMetrics != null) {
-      metricsMap.putAll(additionalQueryMetrics);
-    }
-
-    Gson gson = new GsonBuilder().disableHtmlEscaping().create();
-    return gson.toJson(metricsMap);
   }
 
   protected static Map<String, Serializable> collectQueryProperties(
diff --git a/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/impl/operations/QueryOperationsTest.java b/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/impl/operations/QueryOperationsTest.java
index 69c353cf5932..041b549a7ab6 100644
--- a/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/impl/operations/QueryOperationsTest.java
+++ b/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/impl/operations/QueryOperationsTest.java
@@ -18,15 +18,10 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 
-import ddf.catalog.operation.QueryResponse;
-import ddf.catalog.operation.impl.QueryImpl;
-import ddf.catalog.operation.impl.QueryRequestImpl;
-import ddf.catalog.operation.impl.QueryResponseImpl;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
 import org.junit.Test;
-import org.opengis.filter.Filter;
 
 public class QueryOperationsTest {
 
@@ -112,24 +107,4 @@ public void testCollectQueryMetrics() throws NullPointerException {
         containsString(
             QueryOperations.QM_POSTQUERY + "metric3" + QueryOperations.QM_ELAPSED + ": 98765"));
   }
-
-  @Test
-  public void testLogQueryMetrics() throws NullPointerException {
-    QueryResponse queryResponse =
-        new QueryResponseImpl(new QueryRequestImpl(new QueryImpl(Filter.INCLUDE)));
-    Map<String, Serializable> respProps = new HashMap<>();
-
-    respProps.put(QueryOperations.QM_TRACEID, "12345");
-    respProps.put(QueryOperations.QM_PREQUERY + "metric" + QueryOperations.QM_ELAPSED, "23456");
-    respProps.put(QueryOperations.QM_DO_QUERY + "metric" + QueryOperations.QM_ELAPSED, "23456");
-    respProps.put(
-        QueryOperations.QM_TOTAL_ELAPSED + "metric" + QueryOperations.QM_ELAPSED, "523456");
-    respProps.put(QueryOperations.QM_POSTQUERY + "metric" + QueryOperations.QM_ELAPSED, "45678");
-    respProps.put("metrics-enabled", true);
-    respProps.put("additional-query-metrics", new HashMap<>());
-
-    queryResponse.getProperties().putAll(respProps);
-    String queryMetricsLog = QueryOperations.getQueryMetricsLog(queryResponse.getProperties());
-    assertNotNull(queryMetricsLog);
-  }
 }


Reviews:
user: login: jlcsmith
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-11T21:17:01Z
user: login: derekwilhelm
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T19:27:46Z
user: login: derekwilhelm
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T22:27:09Z
user: login: alexabird
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-20T22:44:26Z

Review comments:
diff_hunk: @@ -209,6 +214,9 @@ QueryResponse query(
       boolean overrideFanoutRename,
       boolean fanoutEnabled)
       throws UnsupportedQueryException, FederationException {
+    DateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
+    format.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String startTime = format.format(new Date());
path: catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/impl/operations/QueryOperations.java
user: login: derekwilhelm
body: ✏️ This would be more concise and wouldn't rely on string values if you changed these lines to:
`String startTime = (new Date()).toInstant().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_DATE_TIME);`
created_at: 2025-03-12T19:20:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -309,6 +319,11 @@ protected static String getQueryMetricsLog(Map<String, Serializable> properties)
     properties.entrySet().stream()
         .filter(e -> e.getKey() != null && e.getKey().startsWith(QM_TIMED_SOURCE))
         .forEach(e -> sourceDurationMap.put(e.getKey().split(QM_TIMED_SOURCE)[1], e.getValue()));
+    Serializable cacheSourceValue = sourceDurationMap.remove("cache");
+    if (cacheSourceValue != null) {
+      String sourceName = (String) properties.get(CACHE_SOURCES_KEY);
+      sourceDurationMap.put(sourceName == null ? "Unknown Source" : sourceName, cacheSourceValue);
+    }
path: catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/impl/operations/QueryOperations.java
user: login: derekwilhelm
body: ❓ ❗  This feels like this should be handled in downstream projects.  In DDF, the source and the cache can be queried together and this will override/replace any information coming from the original source duration statistics.  If a downstream project wants to report the statistics this way, it could do this level of manipulation as a plugin rather than forcing all DDF programs to report the same.  It may be important to know how long the cache query took compared to the original source.  Also, this assumes only a single source was queried in the cache.  That doesn't have to be true so the sources key might be a list of sources, which this code doesn't handle.
created_at: 2025-03-12T19:27:32Z
author_association: CONTRIBUTOR

PR comments:
user: login: zkirksey
created_at: 2025-03-11T01:19:45Z
body: build now
author_association: COLLABORATOR
user: login: cxddfbot
created_at: 2025-03-11T01:20:27Z
body: Internal build has been started, your results will be available at build completion.
author_association: NONE
user: login: cxddfbot
created_at: 2025-03-11T01:26:46Z
body: Build FAILURE See the job results in [legacy Jenkins UI](https://jenkins.phx.connexta.com/job/DDF/job/PR-6870/1/) or in [Blue Ocean UI](https://jenkins.phx.connexta.com/job/DDF/job/PR-6870/1/display/redirect).
author_association: NONE
user: login: alexabird
created_at: 2025-03-11T04:56:09Z
body: <img width="1492" alt="image" src="https://github.com/user-attachments/assets/c1bf6c85-fad1-4caf-86fa-e5a698328c5f" />

author_association: MEMBER
user: login: zkirksey
created_at: 2025-03-11T15:39:49Z
body: build now
author_association: COLLABORATOR
user: login: cxddfbot
created_at: 2025-03-11T15:40:26Z
body: Internal build has been started, your results will be available at build completion.
author_association: NONE
user: login: cxddfbot
created_at: 2025-03-11T16:48:58Z
body: Build SUCCESS See the job results in [legacy Jenkins UI](https://jenkins.phx.connexta.com/job/DDF/job/PR-6870/2/) or in [Blue Ocean UI](https://jenkins.phx.connexta.com/job/DDF/job/PR-6870/2/display/redirect).
author_association: NONE
user: login: zkirksey
created_at: 2025-03-19T21:12:40Z
body: build now
author_association: COLLABORATOR
user: login: cxddfbot
created_at: 2025-03-19T21:13:19Z
body: Internal build has been started, your results will be available at build completion.
author_association: NONE
user: login: cxddfbot
created_at: 2025-03-19T22:33:37Z
body: Build SUCCESS See the job results in [legacy Jenkins UI](https://jenkins.phx.connexta.com/job/DDF/job/PR-6870/1/) or in [Blue Ocean UI](https://jenkins.phx.connexta.com/job/DDF/job/PR-6870/1/display/redirect).
author_association: NONE