Repository: metabase/metabase
Pull request #56451: Add filtering capabilities to `/task/` endpoint and add`/task/unique_tasks` endpoint
Author: login: lbrdnk
Labels: name: no-backport
description: Do not backport this PR to any branch
name: .Team/SemanticLayer
description: DEPRECATED Please use .Team/Querying instead
Created: 2025-04-08T15:49:19Z | Merged: 2025-04-13T07:44:24Z

Closes https://linear.app/metabase/issue/SEM-226/support-filtering-and-sorting-of-the-task-list-in-the-backend

### Description

Adds filtering on status and task to `/task/` endpoint. Adds `/task/unique_tasks/` endpoint.

### How to verify

Run new tests.

### Checklist

- [X] Tests have been added/updated to cover changes in this PR



Diff:
diff --git a/src/metabase/api/task.clj b/src/metabase/api/task.clj
index 71991df24696..8dea7fddd17a 100644
--- a/src/metabase/api/task.clj
+++ b/src/metabase/api/task.clj
@@ -12,12 +12,14 @@
 
 (api.macros/defendpoint :get "/"
   "Fetch a list of recent tasks stored as Task History"
-  []
+  [_
+   {:keys [_status _task]
+    :as filter} :- task-history/Filter]
   (validation/check-has-application-permission :monitoring)
   {:total  (t2/count :model/TaskHistory)
    :limit  (request/limit)
    :offset (request/offset)
-   :data   (task-history/all (request/limit) (request/offset))})
+   :data   (task-history/all (request/limit) (request/offset) filter)})
 
 (api.macros/defendpoint :get "/:id"
   "Get `TaskHistory` entry with ID."
@@ -30,3 +32,11 @@
   []
   (validation/check-has-application-permission :monitoring)
   (task/scheduler-info))
+
+(api.macros/defendpoint :get "/unique-tasks"
+  "Returns possibly empty vector of unique task names in alphabetical order. It is expected that number of unique
+  tasks is small, hence no need for pagination. If that changes this endpoint and function that powers it should
+  reflect that."
+  [] :- [:vector string?]
+  (validation/check-has-application-permission :monitoring)
+  (task-history/unique-tasks))
diff --git a/src/metabase/models/task_history.clj b/src/metabase/models/task_history.clj
index e2b6ca2ef037..160e4881be4a 100644
--- a/src/metabase/models/task_history.clj
+++ b/src/metabase/models/task_history.clj
@@ -64,16 +64,36 @@
   {:task_details mi/transform-json-eliding
    :status       mi/transform-keyword})
 
+(defn- filter->where
+  [{:keys [status task] :as filter}]
+  (when (not-empty filter)
+    {:where (cond-> [:and]
+              task   (conj [:= :task task])
+              status (conj [:= :status (name status)]))}))
+
+(def Filter
+  "Schema for filter for task history."
+  [:maybe [:map [:status {:optional true} (into [:enum] task-history-status)
+                 :task   {:optional true} [:string {:min 1}]]]])
+
 (mu/defn all
   "Return all TaskHistory entries, applying `limit` and `offset` if not nil"
   [limit  :- [:maybe ms/PositiveInt]
-   offset :- [:maybe ms/IntGreaterThanOrEqualToZero]]
+   offset :- [:maybe ms/IntGreaterThanOrEqualToZero]
+   filter :- Filter]
   (t2/select :model/TaskHistory (merge {:order-by [[:started_at :desc]]}
+                                       (filter->where filter)
                                        (when limit
                                          {:limit limit})
                                        (when offset
                                          {:offset offset}))))
 
+(defn unique-tasks
+  "Return _vector_ of all unique tasks' names in alphabetical order."
+  []
+  (vec (t2/select-fn-vec :task [:model/TaskHistory :task] {:group-by [:task]
+                                                           :order-by [:task]})))
+
 ;;; +----------------------------------------------------------------------------------------------------------------+
 ;;; |                                            with-task-history macro                                             |
 ;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/test/metabase/api/task_test.clj b/test/metabase/api/task_test.clj
index 0034c06cac1e..fddc56ba665c 100644
--- a/test/metabase/api/task_test.clj
+++ b/test/metabase/api/task_test.clj
@@ -124,3 +124,199 @@
                  [:jobs      [:sequential
                               [:map-of :any :any]]]]
                 (mt/user-http-request :crowberto :get 200 "task/info")))))
+
+(deftest ^:synchronized single-filter-test
+  (testing "Check that paging information is applied when provided and included in the response"
+    (t2/delete! :model/TaskHistory)
+    (let [now (t/zoned-date-time)]
+      (mt/with-temp [:model/TaskHistory
+                     _
+                     {:status "success"
+                      :task "success"
+                      :started_at (t/minus now (t/hours 1))
+                      :ended_at (t/plus (t/minus now (t/hours 1)) (t/seconds 30))}
+
+                     :model/TaskHistory
+                     _
+                     {:status :failed
+                      :task "failed"
+                      :started_at (t/zoned-date-time)
+                      :ended_at (t/zoned-date-time)}
+
+                     :model/TaskHistory
+                     _
+                     {:status :started
+                      :task "started"
+                      :started_at (t/zoned-date-time)}]
+        (letfn [(status-test-filtering-response
+                  [status]
+                  (testing (format "Filtering for %s response works correctly" status)
+                    (let [response (mt/user-http-request :crowberto :get 200 "task/" :status status)]
+                      (is (= 1 (-> response :data count)))
+                      (is (= status (-> response :data first :task)))
+                      (is (= status (-> response :data first :status))))))]
+          (status-test-filtering-response "success")
+          (status-test-filtering-response "started")
+          (status-test-filtering-response "failed"))
+        (testing "No filter in query params returns all tasks"
+          (let [response (mt/user-http-request :crowberto :get 200 "task/")]
+            (is (= 3 (-> response :data count)))))
+        (testing "Error is returned on explicit nil status"
+          (is (= "enum of :started, :success, :failed"
+                 (-> (mt/user-http-request :crowberto :get 400 "task/" :status nil) :errors :status first))))
+        (testing "Error is returned for unexpected status values"
+          (is (= "enum of :started, :success, :failed"
+                 (-> (mt/user-http-request :crowberto :get 400 "task/" :status 1) :errors :status first))))
+        (letfn [(task-test-filtering-response
+                  [task]
+                  (testing (format "Filtering for `%s` named task works correctly" task)
+                    (let [response (mt/user-http-request :crowberto :get 200 "task/" :task task)]
+                      (is (= 1 (-> response :data count)))
+                      (is (= task (-> response :data first :task)))
+                      (is (= task (-> response :data first :status))))))]
+          (task-test-filtering-response "success")
+          (task-test-filtering-response "started")
+          (task-test-filtering-response "failed"))))))
+
+(deftest ^:synchronized combined-filter-test
+  (testing "Check that paging information is applied when provided and included in the response"
+    (t2/delete! :model/TaskHistory)
+    (let [now (t/zoned-date-time)]
+      (mt/with-temp [:model/TaskHistory
+                     _
+                     {:status "success"
+                      :task "a"
+                      :started_at (t/minus now (t/hours 1))
+                      :ended_at (t/plus (t/minus now (t/hours 1)) (t/seconds 30))}
+
+                     :model/TaskHistory
+                     _
+                     {:status :failed
+                      :task "b"
+                      :started_at (t/zoned-date-time)
+                      :ended_at (t/zoned-date-time)}
+
+                     :model/TaskHistory
+                     _
+                     {:status :started
+                      :task "c"
+                      :started_at (t/zoned-date-time)}]
+        (doseq [task ["a" "b" "c"]
+                status [:success :failed :started]
+                :let [expecting-results (or (and (= task "a") (= status :success))
+                                            (and (= task "b") (= status :failed))
+                                            (and (= task "c") (= status :started)))]]
+          (let [response (mt/user-http-request :crowberto :get 200 "task/" :task task :status status)]
+            (if expecting-results
+              (testing "Correct results returned for combined filter"
+                (is (= 1 (count (:data response))))
+                (is (= task (-> response :data first :task)))
+                (is (= (name status) (-> response :data first :status))))
+              (testing "No results returned for too strict combined filter"
+                (is (empty? (-> response :data)))))))))))
+
+(deftest ^:synchronized multi-results-combined-filter-test
+  (testing "Check that paging information is applied when provided and included in the response"
+    (t2/delete! :model/TaskHistory)
+    (let [now (t/zoned-date-time)]
+      (mt/with-temp
+        [;; task a
+         :model/TaskHistory
+         _
+         {:status "success"
+          :task "a"
+          :started_at (t/minus now (t/hours 1))
+          :ended_at (t/plus (t/minus now (t/hours 1)) (t/seconds 30))}
+         :model/TaskHistory
+         _
+         {:status "failed"
+          :task "a"
+          :started_at (t/minus now (t/hours 1))
+          :ended_at (t/plus (t/minus now (t/hours 1)) (t/seconds 30))}
+
+         ;; task b
+         :model/TaskHistory
+         _
+         {:status :failed
+          :task "b"
+          :started_at (t/zoned-date-time)
+          :ended_at (t/zoned-date-time)}
+         :model/TaskHistory
+         _
+         {:status :started
+          :task "b"
+          :started_at (t/zoned-date-time)}
+
+         ;; task c
+         :model/TaskHistory
+         _
+         {:status :started
+          :task "c"
+          :started_at (t/zoned-date-time)}
+         :model/TaskHistory
+         _
+         {:status :success
+          :task "c"
+          :started_at (t/zoned-date-time)
+          :ended_at (t/zoned-date-time)}]
+        (testing "Multiple tasks returned for task"
+          (let [response (mt/user-http-request :crowberto :get 200 "task/" :task "a")]
+            (is (= 2 (-> response :data count)))
+            (is (some (comp #{"failed"} :status) (-> response :data)))
+            (is (some (comp #{"success"} :status) (-> response :data)))))
+        (testing "Multiple tasks returned for status"
+          (let [response (mt/user-http-request :crowberto :get 200 "task/" :status :failed)]
+            (is (= 2 (-> response :data count)))
+            (is (some (comp #{"b"} :task) (-> response :data)))
+            (is (some (comp #{"a"} :task) (-> response :data)))))))))
+
+(deftest ^:synchronized unique-tasks-test
+  (testing "Check that paging information is applied when provided and included in the response"
+    (t2/delete! :model/TaskHistory)
+    (testing "Empty vector is returned for empty task list"
+      (is (= []
+             (mt/user-http-request :crowberto :get 200 "task/unique-tasks"))))
+    (let [now (t/zoned-date-time)]
+      (mt/with-temp
+        [;; task a
+         :model/TaskHistory
+         _
+         {:status "success"
+          :task "a"
+          :started_at (t/minus now (t/hours 1))
+          :ended_at (t/plus (t/minus now (t/hours 1)) (t/seconds 30))}
+         :model/TaskHistory
+         _
+         {:status "failed"
+          :task "a"
+          :started_at (t/minus now (t/hours 1))
+          :ended_at (t/plus (t/minus now (t/hours 1)) (t/seconds 30))}
+
+           ;; task b
+         :model/TaskHistory
+         _
+         {:status :failed
+          :task "b"
+          :started_at (t/zoned-date-time)
+          :ended_at (t/zoned-date-time)}
+         :model/TaskHistory
+         _
+         {:status :started
+          :task "b"
+          :started_at (t/zoned-date-time)}
+
+           ;; task c
+         :model/TaskHistory
+         _
+         {:status :started
+          :task "c"
+          :started_at (t/zoned-date-time)}
+         :model/TaskHistory
+         _
+         {:status :success
+          :task "c"
+          :started_at (t/zoned-date-time)
+          :ended_at (t/zoned-date-time)}]
+        (testing "Ordered unique tasks are returned"
+          (is (= ["a" "b" "c"]
+                 (mt/user-http-request :crowberto :get 200 "task/unique-tasks"))))))))


Reviews:
user: login: lbrdnk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T08:35:48Z
user: login: piranha
body: Seems good to me, but I'd shorten a bit of code to make it more obvious.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T11:50:21Z
user: login: kamilmielnik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T12:45:42Z

Review comments:
diff_hunk: @@ -64,16 +65,41 @@
   {:task_details mi/transform-json-eliding
    :status       mi/transform-keyword})
 
+(defn- filter->where
+  [{:keys [status task] :as filter}]
+  (when task
+    (assert (string? task)))
+  (when status
+    (assert (keyword? status)))
+  (when (not-empty filter)
+    (let [task-filter (when task [:= :task task])
+          status-filter (when status [:= :status (name status)])]
+      {:where (if (and task-filter status-filter)
+                (conj [:and] task-filter status-filter)
+                (or task-filter status-filter))})))
+
+(mr/def ::filter
path: src/metabase/models/task_history.clj
user: login: lbrdnk
body: Would you find _registering_ this schema reasonable?
created_at: 2025-04-11T08:35:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,3 +32,11 @@
   []
   (validation/check-has-application-permission :monitoring)
   (task/scheduler-info))
+
+(api.macros/defendpoint :get "/unique_tasks"
path: src/metabase/api/task.clj
user: login: piranha
body: While we have both underscores and dashes in our URL, I think the general consensus is on dashes, `/unique-tasks`. :)
created_at: 2025-04-11T08:56:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -64,16 +65,41 @@
   {:task_details mi/transform-json-eliding
    :status       mi/transform-keyword})
 
+(defn- filter->where
+  [{:keys [status task] :as filter}]
+  (when task
+    (assert (string? task)))
+  (when status
+    (assert (keyword? status)))
+  (when (not-empty filter)
+    (let [task-filter (when task [:= :task task])
+          status-filter (when status [:= :status (name status)])]
+      {:where (if (and task-filter status-filter)
+                (conj [:and] task-filter status-filter)
+                (or task-filter status-filter))})))
+
+(mr/def ::filter
path: src/metabase/models/task_history.clj
user: login: piranha
body: I think registering only makes sense when it's re-used somewhere? In this case it's only a few query parameters to a request, so doesn't seem like it makes sense.
created_at: 2025-04-11T11:39:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -64,16 +65,41 @@
   {:task_details mi/transform-json-eliding
    :status       mi/transform-keyword})
 
+(defn- filter->where
+  [{:keys [status task] :as filter}]
+  (when task
+    (assert (string? task)))
+  (when status
+    (assert (keyword? status)))
+  (when (not-empty filter)
+    (let [task-filter (when task [:= :task task])
+          status-filter (when status [:= :status (name status)])]
+      {:where (if (and task-filter status-filter)
+                (conj [:and] task-filter status-filter)
+                (or task-filter status-filter))})))
path: src/metabase/models/task_history.clj
user: login: piranha
body: I'd do something way dumber:
```
(cond-> [:and]
  task (conj [:= :task task])
  status (conj [:= :status status]))
```

a bit more nesting in the result but result is practically the same. Also assertions... idk, malli has checked that already, right?
created_at: 2025-04-11T11:48:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -64,16 +64,36 @@
   {:task_details mi/transform-json-eliding
    :status       mi/transform-keyword})
 
+(defn- filter->where
+  [{:keys [status task] :as filter}]
+  (when (not-empty filter)
+    {:where (cond-> [:and]
+              task   (conj [:= :task task])
+              status (conj [:= :status (name status)]))}))
+
+(def Filter
+  "Schema for filter for task history."
+  [:maybe [:map [:status {:optional true} (into [:enum] task-history-status)
+                 :task   {:optional true} [:string {:min 1}]]]])
+
 (mu/defn all
   "Return all TaskHistory entries, applying `limit` and `offset` if not nil"
path: src/metabase/models/task_history.clj
user: login: kamilmielnik
body: This comment is no longer accurate
created_at: 2025-04-14T11:56:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -12,12 +12,14 @@
 
 (api.macros/defendpoint :get "/"
   "Fetch a list of recent tasks stored as Task History"
-  []
+  [_
+   {:keys [_status _task]
path: src/metabase/api/task.clj
user: login: kamilmielnik
body: According to FE code the status can also be `"unknown"`: https://github.com/metabase/metabase/blob/19e92f3/frontend/src/metabase-types/api/task.ts#L12-L13
created_at: 2025-04-14T11:57:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -12,12 +12,14 @@
 
 (api.macros/defendpoint :get "/"
   "Fetch a list of recent tasks stored as Task History"
-  []
+  [_
+   {:keys [_status _task]
+    :as filter} :- task-history/Filter]
   (validation/check-has-application-permission :monitoring)
   {:total  (t2/count :model/TaskHistory)
    :limit  (request/limit)
    :offset (request/offset)
path: src/metabase/api/task.clj
user: login: kamilmielnik
body: :x: Pagination attributes should take `filter` into account. 

In the screenshot `total` should be `8`:

![image](https://github.com/user-attachments/assets/e0050c61-1ebd-4526-b67c-d7a31e691912)


created_at: 2025-04-14T12:43:06Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-04-08T16:35:53Z
body: <!-- Trunk Test Analytics -->


[![Static Badge](https://img.shields.io/badge/110141_tests-d3e5f11f-purple)](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment) &ensp; [![Static Badge](https://img.shields.io/badge/2-failed-crimson)](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://img.shields.io/badge/2-flaky-red)](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=FAILURE) &ensp; [![Static Badge](https://img.shields.io/badge/0-quarantined-yellow)](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=QUARANTINED)


| Failed Test | Failure Summary | Logs |
| --- | --- | --- |
| <code>later-page-fetch-returns-nil-test</code> |  | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=FAILURE&runId=04fd899d-6f3a-550f-84b9-045a300dc14c) |
| <code>identity-hash-test</code> |  | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=FAILURE&runId=dc61a906-6ec0-5031-8473-92e62f4d84a6) |


| Flaky Test | Failure Summary | Logs |
| --- | --- | --- |
| <code>issue 21452 should not fire POST request after every character during display name change (metabase\#21452)</code> |  | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=FAILURE&runId=8b735d30-4dc3-5181-9063-49221568a9d5) |
| <code>identity-hash-test</code> |  | [Logs ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment&runConclusion=FAILURE&runId=f38913c0-d090-5f99-981e-a2b1c74b8c71) |
<sub>

[View Full Report ↗︎](https://app.trunk.io/metabase/flaky-tests/pr/56451?repo=metabase/metabase&commitHash=d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05&utm_source=gh-comment) ⋅ [Docs](https://docs.trunk.io/flaky-tests?utm_source=gh-comment)

</sub>
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-11T15:25:31Z
body: 
  ## e2e tests failed on `d3e5f11fd95bef06e881cc4452afb3e1cfaf3b05-1`

  [e2e test run](https://github.com/metabase/metabase/actions/runs/14406290834?pr=56451)

  File | Test Name
  ---- | ---------
  `visualizations-charts-reproductions.cy.spec.js` | issue 21452 > should not fire POST request after every character during display name change (metabase#21452)

author_association: CONTRIBUTOR