Repository: checkmarble/marble-backend
Pull request #999: Improved logging for decisions.
Author: login: apognu
Labels: name: enhancement
description: New feature or request
name: go
description: Pull requests that update Go code
Created: 2025-05-14T07:02:55Z | Merged: 2025-05-15T10:48:31Z

 - Decision storage in now measured and logged
 - Each decision, on top of logging the overall evaluation duration now also logs:
   - Each macro-step in the decision process (trigger expression, rules execution, sanctions check, name recognition)
   - Each rule individual duration
 - Add a `LOG_LEVEL` knob (set to `info` or `debug`) to control how verbose JSON and GCP adapters are


Diff:
diff --git a/.env.example b/.env.example
index bea230797..e23bd6e8b 100644
--- a/.env.example
+++ b/.env.example
@@ -98,6 +98,8 @@ CONVOY_PROJECT_ID=
 REQUEST_LOGGING_LEVEL=all
 # 'text', 'json' or 'gcp'
 LOGGING_FORMAT=text
+# 'info' or 'debug'
+# LOG_LEVEL=info
 
 # You can customize the default timeout durations by uncommenting the following settings. Provided values are defaults.
 # BATCH_TIMEOUT_SECOND=55
diff --git a/cmd/server.go b/cmd/server.go
index e375b9b42..679576c57 100644
--- a/cmd/server.go
+++ b/cmd/server.go
@@ -115,6 +115,7 @@ func RunServer(config CompiledConfig) error {
 	}
 
 	logger := utils.NewLogger(serverConfig.loggingFormat)
+
 	ctx := utils.StoreLoggerInContext(context.Background(), logger)
 	marbleJwtSigningKey := infra.ReadParseOrGenerateSigningKey(ctx, serverConfig.jwtSigningKey, serverConfig.jwtSigningKeyFile)
 	license := infra.VerifyLicense(licenseConfig)
diff --git a/models/decision.go b/models/decision.go
index f83248fd1..ef041a37f 100644
--- a/models/decision.go
+++ b/models/decision.go
@@ -79,6 +79,13 @@ type ScenarioExecution struct {
 	Outcome                Outcome
 	OrganizationId         string
 	TestRunId              string
+
+	ExecutionMetrics *ScenarioExecutionMetrics
+}
+
+type ScenarioExecutionMetrics struct {
+	Steps map[string]int64
+	Rules map[string]int64
 }
 
 type RuleExecutionStat struct {
@@ -97,6 +104,7 @@ type RuleExecution struct {
 	Result              bool
 	ResultScoreModifier int
 	Rule                Rule
+	Duration            time.Duration
 }
 
 func AdaptScenarExecToDecision(scenarioExecution ScenarioExecution, clientObject ClientObject, scheduledExecutionId *string) DecisionWithRuleExecutions {
diff --git a/models/sanction_check.go b/models/sanction_check.go
index 153c58197..32305137d 100644
--- a/models/sanction_check.go
+++ b/models/sanction_check.go
@@ -119,6 +119,9 @@ type SanctionCheckWithMatches struct {
 	SanctionCheck
 	Matches []SanctionCheckMatch
 	Count   int
+
+	Duration                time.Duration
+	NameRecognitionDuration time.Duration
 }
 
 type SanctionRawSearchResponseWithMatches struct {
diff --git a/pure_utils/map.go b/pure_utils/map.go
index 3ede93326..7fda118b5 100644
--- a/pure_utils/map.go
+++ b/pure_utils/map.go
@@ -108,3 +108,14 @@ func MapKeyValue[KL, KR comparable, VL, VR any](in map[KL]VL, f func(k KL, v VL)
 
 	return out
 }
+
+func MapSliceToMap[T, V any, K comparable](input []T, f func(v T) (K, V)) map[K]V {
+	output := make(map[K]V, len(input))
+
+	for _, item := range input {
+		k, v := f(item)
+		output[k] = v
+	}
+
+	return output
+}
diff --git a/usecases/custom_list_usecase.go b/usecases/custom_list_usecase.go
index 2464326a3..2e2d17681 100644
--- a/usecases/custom_list_usecase.go
+++ b/usecases/custom_list_usecase.go
@@ -231,7 +231,7 @@ func (usecase *CustomListUseCase) ReplaceCustomListValuesFromCSV(ctx context.Con
 		duration := end.Sub(start)
 		// divide by 1e6 convert to milliseconds (base is nanoseconds)
 		avgDuration := float64(duration) / float64(total*1e6)
-		logger.InfoContext(ctx, fmt.Sprintf("Successfully ingested %d custom list values in %s, average %vms", total, duration, avgDuration))
+		logger.DebugContext(ctx, fmt.Sprintf("Successfully ingested %d custom list values in %s, average %vms", total, duration, avgDuration))
 	}
 	defer printDuration()
 
diff --git a/usecases/decision_usecase.go b/usecases/decision_usecase.go
index 64fc38db5..f197a3613 100644
--- a/usecases/decision_usecase.go
+++ b/usecases/decision_usecase.go
@@ -367,6 +367,8 @@ func (usecase *DecisionUsecase) CreateDecision(
 	input models.CreateDecisionInput,
 	params models.CreateDecisionParams,
 ) (bool, models.DecisionWithRuleExecutions, error) {
+	decisionStart := time.Now()
+
 	exec := usecase.executorFactory.NewExecutor()
 	tracer := utils.OpenTelemetryTracerFromContext(ctx)
 	ctx, span := tracer.Start(
@@ -441,6 +443,7 @@ func (usecase *DecisionUsecase) CreateDecision(
 	defer span.End()
 
 	sendWebhookEventId := make([]string, 0)
+	storageStart := time.Now()
 	newDecision, err := executor_factory.TransactionReturnValue(ctx, usecase.transactionFactory, func(
 		tx repositories.Transaction,
 	) (models.DecisionWithRuleExecutions, error) {
@@ -517,6 +520,25 @@ func (usecase *DecisionUsecase) CreateDecision(
 		return false, models.DecisionWithRuleExecutions{}, err
 	}
 
+	if scenarioExecution.ExecutionMetrics != nil {
+		storageDuration := time.Since(storageStart)
+		decisionDuration := time.Since(decisionStart)
+
+		scenarioExecution.ExecutionMetrics.Steps[evaluate_scenario.LogStorageDurationKey] = storageDuration.Milliseconds()
+
+		utils.LoggerFromContext(ctx).InfoContext(ctx,
+			fmt.Sprintf("created decision %s in %dms", decision.DecisionId, decisionDuration.Milliseconds()),
+			"org_id", scenario.OrganizationId,
+			"decision_id", decision.DecisionId,
+			"scenario_id", scenario.Id,
+			"score", scenarioExecution.Score,
+			"outcome", scenarioExecution.Outcome,
+			"duration", decisionDuration.Milliseconds(),
+			"rules", scenarioExecution.ExecutionMetrics.Rules,
+			"steps", scenarioExecution.ExecutionMetrics.Steps)
+
+	}
+
 	for _, webhookEventId := range sendWebhookEventId {
 		usecase.webhookEventsSender.SendWebhookEventAsync(ctx, webhookEventId)
 	}
@@ -530,6 +552,7 @@ func (usecase *DecisionUsecase) CreateAllDecisions(
 	ctx context.Context,
 	input models.CreateAllDecisionsInput,
 ) (decisions []models.DecisionWithRuleExecutions, nbSkipped int, err error) {
+	decisionStart := time.Now()
 	exec := usecase.executorFactory.NewExecutor()
 	tracer := utils.OpenTelemetryTracerFromContext(ctx)
 	ctx, span := tracer.Start(ctx, "DecisionUsecase.CreateAllDecisions")
@@ -571,8 +594,9 @@ func (usecase *DecisionUsecase) CreateAllDecisions(
 	}
 
 	type decisionAndScenario struct {
-		decision models.DecisionWithRuleExecutions
-		scenario models.Scenario
+		decision  models.DecisionWithRuleExecutions
+		scenario  models.Scenario
+		execution models.ScenarioExecution
 	}
 	var items []decisionAndScenario
 	for _, scenario := range filteredScenarios {
@@ -602,7 +626,10 @@ func (usecase *DecisionUsecase) CreateAllDecisions(
 			usecase.executeTestRun(ctx, input.OrganizationId, input.TriggerObjectTable, evaluationParameters, scenario, nil)
 		default:
 			decision := models.AdaptScenarExecToDecision(scenarioExecution, payload, nil)
-			items = append(items, decisionAndScenario{decision: decision, scenario: scenario})
+			items = append(items, decisionAndScenario{
+				decision: decision,
+				scenario: scenario, execution: scenarioExecution,
+			})
 			usecase.executeTestRun(ctx, input.OrganizationId, input.TriggerObjectTable,
 				evaluationParameters, scenario, &scenarioExecution)
 		}
@@ -618,6 +645,7 @@ func (usecase *DecisionUsecase) CreateAllDecisions(
 		var ids []string
 		for _, item := range items {
 			ids = append(ids, item.decision.DecisionId)
+			storageStart := time.Now()
 			if err = usecase.repository.StoreDecision(
 				ctx,
 				tx,
@@ -644,6 +672,25 @@ func (usecase *DecisionUsecase) CreateAllDecisions(
 				}
 			}
 
+			if item.execution.ExecutionMetrics != nil {
+				storageDuration := time.Since(storageStart)
+				decisionDuration := time.Since(decisionStart)
+
+				item.execution.ExecutionMetrics.Steps[evaluate_scenario.LogStorageDurationKey] = storageDuration.Milliseconds()
+
+				utils.LoggerFromContext(ctx).InfoContext(ctx,
+					fmt.Sprintf("created decision (all) %s in %dms",
+						item.decision.DecisionId, decisionDuration.Milliseconds()),
+					"org_id", item.scenario.OrganizationId,
+					"decision_id", item.decision.DecisionId,
+					"scenario_id", item.scenario.Id,
+					"score", item.execution.Score,
+					"outcome", item.execution.Outcome,
+					"duration", decisionDuration.Milliseconds(),
+					"rules", item.execution.ExecutionMetrics.Rules,
+					"steps", item.execution.ExecutionMetrics.Steps)
+			}
+
 			webhookEventId := uuid.NewString()
 			err := usecase.webhookEventsSender.CreateWebhookEvent(ctx, tx, models.WebhookEventCreate{
 				Id:             webhookEventId,
diff --git a/usecases/evaluate_scenario/evaluate_sanction_check.go b/usecases/evaluate_scenario/evaluate_sanction_check.go
index c9d405386..449d3d59c 100644
--- a/usecases/evaluate_scenario/evaluate_sanction_check.go
+++ b/usecases/evaluate_scenario/evaluate_sanction_check.go
@@ -3,6 +3,7 @@ package evaluate_scenario
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/checkmarble/marble-backend/models"
 	"github.com/cockroachdb/errors"
@@ -27,6 +28,8 @@ func (e ScenarioEvaluator) evaluateSanctionCheck(
 		return
 	}
 
+	start := time.Now()
+
 	if iteration.SanctionCheckConfig.TriggerRule != nil {
 		triggerEvaluation, err := e.evaluateAstExpression.EvaluateAstExpression(
 			ctx,
@@ -64,14 +67,20 @@ func (e ScenarioEvaluator) evaluateSanctionCheck(
 		}
 	}
 
+	var nameRecognitionDuration time.Duration
+
 	if iteration.SanctionCheckConfig.Query.Label != nil {
 		nbEvaluatedFields += 1
-		queries, emptyInput, err = e.evaluateSanctionCheckLabel(ctx, queries, iteration, dataAccessor)
+		nameRecognizedQueries, emptyInput, duration, err :=
+			e.evaluateSanctionCheckLabel(ctx, queries, iteration, dataAccessor)
 		if err != nil {
 			return nil, true, errors.Wrap(err, "could not evaluate sanction check label")
 		} else if emptyInput {
 			emptyFieldRead += 1
 		}
+
+		queries = nameRecognizedQueries
+		nameRecognitionDuration = duration
 	}
 
 	if emptyFieldRead == nbEvaluatedFields {
@@ -146,6 +155,8 @@ func (e ScenarioEvaluator) evaluateSanctionCheck(
 	}
 
 	sanctionCheck = &result
+	sanctionCheck.Duration = time.Since(start)
+	sanctionCheck.NameRecognitionDuration = nameRecognitionDuration
 	performed = true
 	return
 }
@@ -192,7 +203,7 @@ func (e ScenarioEvaluator) evaluateSanctionCheckLabel(
 	queries []models.OpenSanctionsCheckQuery,
 	iteration models.ScenarioIteration,
 	dataAccessor DataAccessor,
-) (queriesOut []models.OpenSanctionsCheckQuery, emptyInput bool, err error) {
+) (queriesOut []models.OpenSanctionsCheckQuery, emptyInput bool, took time.Duration, err error) {
 	queriesOut = queries
 	labelFilterAny, err := e.evaluateAstExpression.EvaluateAstExpression(ctx, nil,
 		*iteration.SanctionCheckConfig.Query.Label, iteration.OrganizationId,
@@ -207,7 +218,7 @@ func (e ScenarioEvaluator) evaluateSanctionCheckLabel(
 
 	labelFilter, ok := labelFilterAny.ReturnValue.(string)
 	if !ok {
-		return nil, false, errors.New("label filter name query did not return a string")
+		return nil, false, 0, errors.New("label filter name query did not return a string")
 	}
 	if labelFilter == "" {
 		emptyInput = true
@@ -228,12 +239,14 @@ func (e ScenarioEvaluator) evaluateSanctionCheckLabel(
 			queriesOut[0].Filters["name"] = append(queriesOut[0].Filters["name"], labelFilter)
 		}
 
-		return queriesOut, false, nil
+		return queriesOut, false, 0, nil
 	}
 
+	beforeNameRecognition := time.Now()
+
 	matches, err := e.nameRecognizer.PerformNameRecognition(ctx, labelFilter)
 	if err != nil {
-		return queriesOut, false, errors.Wrap(err,
+		return queriesOut, false, 0, errors.Wrap(err,
 			"could not perform name recognition on label")
 	}
 
@@ -272,5 +285,5 @@ func (e ScenarioEvaluator) evaluateSanctionCheckLabel(
 		queriesOut = append(queriesOut, *companyQuery)
 	}
 
-	return queriesOut, false, nil
+	return queriesOut, false, time.Since(beforeNameRecognition), nil
 }
diff --git a/usecases/evaluate_scenario/evaluate_scenario.go b/usecases/evaluate_scenario/evaluate_scenario.go
index 0196a3f4e..1b323ec44 100644
--- a/usecases/evaluate_scenario/evaluate_scenario.go
+++ b/usecases/evaluate_scenario/evaluate_scenario.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/checkmarble/marble-backend/models"
 	"github.com/checkmarble/marble-backend/models/ast"
+	"github.com/checkmarble/marble-backend/pure_utils"
 	"github.com/checkmarble/marble-backend/repositories"
 	"github.com/checkmarble/marble-backend/repositories/httpmodels"
 	"github.com/checkmarble/marble-backend/usecases/ast_eval"
@@ -25,6 +26,15 @@ import (
 // TODO: set value from configuration/env instead
 const MAX_CONCURRENT_RULE_EXECUTIONS = 5
 
+const (
+	LogTriggerDurationKey    = "trigger"
+	LogEvaluationDurationKey = "evaluation"
+	LogRulesDurationKey      = "rules"
+	LogScreeningDurationKey  = "screening"
+	LogNerDurationKey        = "ner"
+	LogStorageDurationKey    = "storage"
+)
+
 type ScenarioEvaluationParameters struct {
 	Scenario            models.Scenario
 	TargetIterationId   *string
@@ -139,6 +149,9 @@ func (e ScenarioEvaluator) processScenarioIteration(
 
 	// Evaluate the trigger
 
+	beforeTriggerExpression := time.Now()
+	triggerExpressionDuration := 0 * time.Millisecond
+
 	if iteration.TriggerConditionAstExpression != nil {
 		ok, err := e.evalScenarioTrigger(
 			ctx,
@@ -155,6 +168,8 @@ func (e ScenarioEvaluator) processScenarioIteration(
 		if !ok {
 			return false, models.ScenarioExecution{}, nil
 		}
+
+		triggerExpressionDuration = time.Since(beforeTriggerExpression)
 	}
 
 	var pivotValue *string
@@ -184,6 +199,9 @@ func (e ScenarioEvaluator) processScenarioIteration(
 			errSnooze,
 			"error when listing active rule snozze")
 	}
+
+	beforeRules := time.Now()
+
 	// Evaluate all rules
 	score, ruleExecutions, errEval := e.evalAllScenarioRules(
 		ctx,
@@ -197,6 +215,8 @@ func (e ScenarioEvaluator) processScenarioIteration(
 			"error during concurrent rule evaluation")
 	}
 
+	rulesDuration := time.Since(beforeRules)
+
 	var outcome models.Outcome
 
 	sanctionCheckExecution, santionCheckPerformed, err :=
@@ -243,8 +263,34 @@ func (e ScenarioEvaluator) processScenarioIteration(
 	}
 
 	elapsed := time.Since(start)
-	logger.InfoContext(ctx, fmt.Sprintf("Evaluated scenario in %dms",
-		elapsed.Milliseconds()), "score", score, "outcome", outcome, "duration", elapsed.Milliseconds())
+
+	ruleDurations := pure_utils.MapSliceToMap(ruleExecutions, func(exec models.RuleExecution) (string, int64) {
+		id := exec.Rule.Id
+		if exec.Rule.StableRuleId != nil {
+			id = *exec.Rule.StableRuleId
+		}
+
+		return id, exec.Duration.Milliseconds()
+	})
+
+	stepDurations := make(map[string]int64, 5)
+	stepDurations[LogTriggerDurationKey] = triggerExpressionDuration.Milliseconds()
+	stepDurations[LogEvaluationDurationKey] = elapsed.Milliseconds()
+	stepDurations[LogRulesDurationKey] = rulesDuration.Milliseconds()
+
+	if sanctionCheckExecution != nil {
+		stepDurations[LogScreeningDurationKey] = sanctionCheckExecution.Duration.Milliseconds()
+
+		if sanctionCheckExecution.NameRecognitionDuration != 0 {
+			stepDurations[LogNerDurationKey] =
+				sanctionCheckExecution.NameRecognitionDuration.Milliseconds()
+		}
+	}
+
+	se.ExecutionMetrics = &models.ScenarioExecutionMetrics{
+		Steps: stepDurations,
+		Rules: ruleDurations,
+	}
 
 	return true, se, nil
 }
@@ -268,7 +314,7 @@ func (e ScenarioEvaluator) EvalTestRunScenario(
 			se = models.ScenarioExecution{}
 		}
 	}()
-	logger.InfoContext(ctx, "Evaluating scenario test run", "scenarioId", params.Scenario.Id)
+	logger.DebugContext(ctx, "Evaluating scenario test run", "scenarioId", params.Scenario.Id)
 	exec := e.executorFactory.NewExecutor()
 	tracer := utils.OpenTelemetryTracerFromContext(ctx)
 	ctx, span := tracer.Start(ctx, "evaluate_scenario.EvalTestRunScenario",
@@ -349,7 +395,7 @@ func (e ScenarioEvaluator) EvalScenario(
 		}
 	}()
 
-	logger.InfoContext(ctx, "Evaluating scenario", "scenarioId", params.Scenario.Id)
+	logger.DebugContext(ctx, "Evaluating scenario", "scenarioId", params.Scenario.Id)
 	exec := e.executorFactory.NewExecutor()
 
 	// If the scenario has no live version, don't try to Eval() it, return early
@@ -413,6 +459,7 @@ func (e ScenarioEvaluator) evalScenarioRule(
 	dataModel models.DataModel,
 	snoozes []models.RuleSnooze,
 ) (int, models.RuleExecution, error) {
+	start := time.Now()
 	ruleExecution := models.RuleExecution{}
 	tracer := utils.OpenTelemetryTracerFromContext(ctx)
 	ctx, span := tracer.Start(ctx, "evaluate_scenario.evalScenarioRule",
@@ -490,26 +537,24 @@ func (e ScenarioEvaluator) evalScenarioRule(
 		)
 	}
 
+	ruleStats := ast.BuildEvaluationStats(ruleEvaluation, false)
+	functionStats := ruleStats.FunctionStats()
+
 	// Increment scenario score when rule result is true
 	if ruleExecution.Result {
 		ruleExecution.Outcome = "hit"
 		ruleExecution.ResultScoreModifier = rule.ScoreModifier
-		logger.InfoContext(ctx, "Rule executed",
-			slog.Int("score_modifier", rule.ScoreModifier),
-			slog.String("ruleName", rule.Name),
-			slog.Bool("result", ruleExecution.Result),
-		)
-	}
 
-	ruleStats := ast.BuildEvaluationStats(ruleEvaluation, false)
-	functionStats := ruleStats.FunctionStats()
+		logger.DebugContext(ctx, fmt.Sprintf("rule evaluated in %dms",
+			ruleStats.Took.Milliseconds()), "duration",
+			ruleStats.Took.Milliseconds(), "nodes", ruleStats.Nodes, "skipped", ruleStats.SkippedCount,
+			"cached", ruleStats.CachedCount, "ruleName", rule.Name, "score_modifier",
+			rule.ScoreModifier, "result", ruleExecution.Result)
 
-	logger.InfoContext(ctx, fmt.Sprintf("rule evaluated in %dms",
-		ruleStats.Took.Milliseconds()), "duration",
-		ruleStats.Took.Milliseconds(), "nodes", ruleStats.Nodes, "skipped", ruleStats.SkippedCount,
-		"cached", ruleStats.CachedCount)
+		logger.DebugContext(ctx, "rule nodes breakdown", "functions", functionStats)
+	}
 
-	logger.DebugContext(ctx, "rule nodes breakdown", "functions", functionStats)
+	ruleExecution.Duration = time.Since(start)
 
 	return ruleExecution.ResultScoreModifier, ruleExecution, nil
 }
diff --git a/usecases/ingestion_usecase.go b/usecases/ingestion_usecase.go
index 63c8c7328..d2f795a72 100644
--- a/usecases/ingestion_usecase.go
+++ b/usecases/ingestion_usecase.go
@@ -103,7 +103,7 @@ func (usecase *IngestionUseCase) IngestObject(
 		return 0, err
 	}
 
-	logger.InfoContext(ctx, fmt.Sprintf("Successfully ingested objects: %d objects", nb),
+	logger.DebugContext(ctx, fmt.Sprintf("Successfully ingested objects: %d objects", nb),
 		slog.String("organization_id", organizationId),
 		slog.String("object_type", objectType),
 		slog.Int("nb_objects", nb),
@@ -193,7 +193,7 @@ func (usecase *IngestionUseCase) IngestObjects(
 		return 0, err
 	}
 
-	logger.InfoContext(ctx, fmt.Sprintf("Successfully ingested objects: %d objects", nb),
+	logger.DebugContext(ctx, fmt.Sprintf("Successfully ingested objects: %d objects", nb),
 		slog.String("organization_id", organizationId),
 		slog.String("object_type", objectType),
 		slog.Int("nb_objects", nb),
@@ -493,7 +493,7 @@ func (usecase *IngestionUseCase) ingestObjectsFromCSV(ctx context.Context, organ
 		// divide by 1e6 convert to milliseconds (base is nanoseconds)
 		avgDuration := float64(duration) / float64(total*1e6)
 		if total > 0 {
-			logger.InfoContext(ctx, fmt.Sprintf("Successfully ingested %d objects in %s, average %vms", total, duration, avgDuration))
+			logger.DebugContext(ctx, fmt.Sprintf("Successfully ingested %d objects in %s, average %vms", total, duration, avgDuration))
 		}
 	}
 	defer printDuration()
diff --git a/usecases/scheduled_execution/async_decision_job.go b/usecases/scheduled_execution/async_decision_job.go
index fbf80198c..678c3326e 100644
--- a/usecases/scheduled_execution/async_decision_job.go
+++ b/usecases/scheduled_execution/async_decision_job.go
@@ -224,6 +224,7 @@ func (w *AsyncDecisionWorker) createSingleDecisionForObjectId(
 	args models.AsyncDecisionArgs,
 	tx repositories.Transaction,
 ) (decisionCreated bool, webhookIds []string, testRunCallback func(), err error) {
+	decisionStart := time.Now()
 	tracer := utils.OpenTelemetryTracerFromContext(ctx)
 	ctx, span := tracer.Start(
 		ctx,
@@ -336,6 +337,7 @@ func (w *AsyncDecisionWorker) createSingleDecisionForObjectId(
 
 	decision := models.AdaptScenarExecToDecision(scenarioExecution, object, &args.ScheduledExecutionId)
 	sendWebhookEventId := make([]string, 0, 2)
+	storageStart := time.Now()
 
 	err = w.decisionRepository.StoreDecision(
 		ctx,
@@ -348,6 +350,24 @@ func (w *AsyncDecisionWorker) createSingleDecisionForObjectId(
 		return false, nil, nil, errors.Wrapf(err, "error storing decision in AsyncDecisionWorker %s", scenario.Id)
 	}
 
+	if scenarioExecution.ExecutionMetrics != nil {
+		storageDuration := time.Since(storageStart)
+		decisionDuration := time.Since(decisionStart)
+
+		scenarioExecution.ExecutionMetrics.Steps[evaluate_scenario.LogStorageDurationKey] = storageDuration.Milliseconds()
+
+		utils.LoggerFromContext(ctx).InfoContext(ctx,
+			fmt.Sprintf("created decision (async) %s in %dms", decision.DecisionId, decisionDuration.Milliseconds()),
+			"org_id", scenario.OrganizationId,
+			"decision_id", decision.DecisionId,
+			"scenario_id", scenario.Id,
+			"score", scenarioExecution.Score,
+			"outcome", scenarioExecution.Outcome,
+			"duration", decisionDuration.Milliseconds(),
+			"rules", scenarioExecution.ExecutionMetrics.Rules,
+			"steps", scenarioExecution.ExecutionMetrics.Steps)
+	}
+
 	if decision.SanctionCheckExecution != nil {
 		_, err := w.sanctionCheckRepository.InsertSanctionCheck(ctx, tx,
 			decision.DecisionId, *decision.SanctionCheckExecution, true)
diff --git a/usecases/webhook_events_usecase.go b/usecases/webhook_events_usecase.go
index a8258f175..90f5683be 100644
--- a/usecases/webhook_events_usecase.go
+++ b/usecases/webhook_events_usecase.go
@@ -213,7 +213,7 @@ func (usecase WebhookEventsUsecase) _sendWebhookEvent(ctx context.Context, webho
 		return models.Scheduled, err
 	}
 
-	logger.InfoContext(ctx, fmt.Sprintf("Start processing webhook event %s", webhookEvent.Id))
+	logger.DebugContext(ctx, fmt.Sprintf("Start processing webhook event %s", webhookEvent.Id))
 
 	webhookEventUpdate := models.WebhookEventUpdate{Id: webhookEvent.Id}
 
diff --git a/utils/context_logging.go b/utils/context_logging.go
index 65b54a6b1..1526e1396 100644
--- a/utils/context_logging.go
+++ b/utils/context_logging.go
@@ -18,21 +18,32 @@ func NewLogger(format string) *slog.Logger {
 		format = "text"
 	}
 
+	lvl := slog.LevelInfo
+
+	switch {
+	case format == "text" || os.Getenv("LOG_LEVEL") == "debug":
+		lvl = slog.LevelDebug
+	}
+
+	loggerOptions := slog.HandlerOptions{
+		Level: lvl,
+	}
+
 	switch format {
 	case "text":
 		logHandler := LocalDevHandlerOptions{
-			SlogOpts: slog.HandlerOptions{Level: slog.LevelDebug},
+			SlogOpts: loggerOptions,
 			UseColor: true,
 		}.NewLocalDevHandler(os.Stdout)
 		logger = slog.New(logHandler)
 	case "json":
-		logger = slog.New(slog.NewJSONHandler(os.Stdout, nil))
+		logger = slog.New(slog.NewJSONHandler(os.Stdout, &loggerOptions))
 	case "gcp":
 		creds, err := google.FindDefaultCredentials(context.Background())
 		if err != nil || creds.ProjectID == "" {
 			fmt.Printf("failed to find default credentials (%v) or projectId empty: the traceId in logs will not be usable\n", err)
 		}
-		logger = slog.New(NewGcpHandler(creds.ProjectID))
+		logger = slog.New(NewGcpHandler(creds.ProjectID, loggerOptions))
 	}
 	return logger
 }
diff --git a/utils/logging.go b/utils/logging.go
index 641216623..ba6799511 100644
--- a/utils/logging.go
+++ b/utils/logging.go
@@ -174,8 +174,8 @@ func (h *GcpHandler) Handle(ctx context.Context, r slog.Record) error {
 	return h.internalHandler.Handle(ctx, r)
 }
 
-func NewGcpHandler(projectId string) *GcpHandler {
-	slogOption := slog.HandlerOptions{ReplaceAttr: GCPLoggerAttributeReplacer}
+func NewGcpHandler(projectId string, slogOption slog.HandlerOptions) *GcpHandler {
+	slogOption.ReplaceAttr = GCPLoggerAttributeReplacer
 	jsonHandler := slog.NewJSONHandler(os.Stdout, &slogOption)
 	return &GcpHandler{projectId: projectId, internalHandler: jsonHandler}
 }


Reviews:
user: login: Pascal-Delange
body: nice 👍 
I do think it would be great to tune down the highest volume logs to debug, and this PR would be a great place to do so - mostly I'm thinking of the following ones:
- all those logs about rule execution and scenario execution
- ingestion logs about "XX objects ingested"
- "send webhook" logs
- "do test run decision" logs

But really the easiest way to find them would be to look at the logs explorer and filter all stdout logs on the backend.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T19:50:17Z
user: login: apognu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T20:24:55Z
user: login: Pascal-Delange
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T07:19:55Z

Review comments:
diff_hunk: @@ -517,6 +520,22 @@ func (usecase *DecisionUsecase) CreateDecision(
 		return false, models.DecisionWithRuleExecutions{}, err
 	}
 
+	if scenarioExecution.ExecutionMetrics != nil {
+		storageDuration := time.Since(storageStart)
+		decisionDuration := time.Since(decisionStart)
+
+		scenarioExecution.ExecutionMetrics.Steps[evaluate_scenario.LogStorageDurationKey] = storageDuration.Milliseconds()
+
+		utils.LoggerFromContext(ctx).InfoContext(ctx,
+			fmt.Sprintf("created decision %s in %dms", decision.DecisionId,
+				decisionDuration.Milliseconds()), "org_id", scenario.OrganizationId, "decision_id",
+			decision.DecisionId, "scenario_id", scenario.Id, "score", scenarioExecution.Score,
path: usecases/decision_usecase.go
user: login: Pascal-Delange
body: purely for my personal OCD, I'd love one line per key/value :D 
created_at: 2025-05-14T19:46:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -517,6 +520,22 @@ func (usecase *DecisionUsecase) CreateDecision(
 		return false, models.DecisionWithRuleExecutions{}, err
 	}
 
+	if scenarioExecution.ExecutionMetrics != nil {
+		storageDuration := time.Since(storageStart)
+		decisionDuration := time.Since(decisionStart)
+
+		scenarioExecution.ExecutionMetrics.Steps[evaluate_scenario.LogStorageDurationKey] = storageDuration.Milliseconds()
+
+		utils.LoggerFromContext(ctx).InfoContext(ctx,
+			fmt.Sprintf("created decision %s in %dms", decision.DecisionId,
+				decisionDuration.Milliseconds()), "org_id", scenario.OrganizationId, "decision_id",
+			decision.DecisionId, "scenario_id", scenario.Id, "score", scenarioExecution.Score,
path: usecases/decision_usecase.go
user: login: apognu
body: Oh thank God, me too. I didn't want to upset the auto-formatter, but perfect.
created_at: 2025-05-14T20:24:55Z
author_association: CONTRIBUTOR

PR comments:
user: login: apognu
created_at: 2025-05-14T09:55:24Z
body: Maybe this is the time to decode if some of our current `INFO` entries could be set as `DEBUG` instead.
author_association: CONTRIBUTOR
user: login: apognu
created_at: 2025-05-14T20:31:43Z
body: Yes. I also started looking at production logs to see what we could cut on. Glad we agree, I'll append a commit transitioning some of those infos into debugs. 💰
author_association: CONTRIBUTOR