Repository: CSCfi/rems
Pull request #3324 (2024-08-22 06:42:56)

Many handlers perf2.1
Replaces #3313 

Continues work in #3305 

Includes:
- Makes sure handled applications list is fetched lazily so correct amount of work is done when a limit is given (e.g. only 50).
- Logs test data creation speed to be able to easily follow it. This gives an indication of general REMS performance.
- Emails for one application were sent one at a time. E.g., with 12 handlers there would 12 separate calls to save the email into the outbox. Now this is done with a single call, which helps the 100 handler "worst-case scenario". Tests needed updating to use the new function.
- Replaced `mapcat` with other code to make it faster (micro-optimization).
- The main problem for the performance of the `/api/applications/handled` is actually the verification of the return Schema. If there are thousands of applications, there is a lot of data and half of the time is just spent verifying that it matches our Schema. We could investigate if we can leave the checking out in some cases (e.g., when we know there are a lot of applications). See [issue](https://github.com/orgs/CSCfi/projects/13/views/1?pane=issue&itemId=75940840)

# Checklist for author

Remove items that aren't applicable, check items that are done.

## Reviewability
- [x] Link to issue

## Backwards compatibility
- [x] API is backwards compatible or completely new

## Documentation
- [ ] Update changelog if necessary  (no need for a new one as the last improvement is already mentioned)

## Testing
- [ ] Complex logic is unit tested
- [x] Valuable features are integration / browser / acceptance tested automatically

## Follow-up
- [x] New tasks are created for pending or remaining tasks (see [issue](https://github.com/orgs/CSCfi/projects/13/views/1?pane=issue&itemId=75940840))



Patch:
diff --git a/package-lock.json b/package-lock.json
index 440a410e9..be4759319 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -858,12 +858,12 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
-      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
       "dev": true,
       "dependencies": {
-        "fill-range": "^7.1.1"
+        "fill-range": "^7.0.1"
       },
       "engines": {
         "node": ">=8"
@@ -1440,9 +1440,9 @@
       "peer": true
     },
     "node_modules/elliptic": {
-      "version": "6.5.7",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.7.tgz",
-      "integrity": "sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q==",
+      "version": "6.5.4",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz",
+      "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==",
       "dev": true,
       "dependencies": {
         "bn.js": "^4.11.9",
@@ -1579,9 +1579,9 @@
       "dev": true
     },
     "node_modules/fill-range": {
-      "version": "7.1.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
-      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
       "dev": true,
       "dependencies": {
         "to-regex-range": "^5.0.1"
@@ -3045,9 +3045,9 @@
       "dev": true
     },
     "node_modules/shadow-cljs/node_modules/ws": {
-      "version": "7.5.10",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
-      "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==",
+      "version": "7.5.8",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.8.tgz",
+      "integrity": "sha512-ri1Id1WinAX5Jqn9HejiGb8crfRio0Qgu8+MtL36rlTA6RLsMdWt1Az/19A2Qij6uSHUMphEFaTKa4WG+UNHNw==",
       "dev": true,
       "engines": {
         "node": ">=8.3.0"
diff --git a/resources/sql/queries.sql b/resources/sql/queries.sql
index 29dd86df3..15f5ad2d9 100644
--- a/resources/sql/queries.sql
+++ b/resources/sql/queries.sql
@@ -663,6 +663,11 @@ INSERT INTO outbox (outboxData)
 VALUES (:outboxdata::jsonb)
 RETURNING id;
 
+-- :name puts-to-outbox! :returning-execute
+INSERT INTO outbox (outboxData)
+SELECT * FROM jsonb_to_recordset(:outboxdatas::jsonb) AS tmp(outboxdata jsonb)
+RETURNING id;
+
 -- :name get-outbox :? :*
 SELECT id, outboxData::text
 FROM outbox
diff --git a/src/clj/rems/api/applications.clj b/src/clj/rems/api/applications.clj
index 4fbfc241a..ef06b8e42 100644
--- a/src/clj/rems/api/applications.clj
+++ b/src/clj/rems/api/applications.clj
@@ -78,7 +78,7 @@
 (defn- overview-only-active-handlers [app]
   (update-in app
              [:application/workflow :workflow.dynamic/handlers]
-             #(filterv :handler/active? %)))
+             #(filter :handler/active? %)))
 
 ;; Api implementation
 
@@ -135,6 +135,14 @@
     (merge {:success true}
            (commands/validate-application application (:field-values request)))))
 
+(defn- get-handled-applications [{:keys [query only-active-handlers limit]}]
+  (time
+   (cond->> (todos/get-handled-todos (getx-user-id))
+     only-active-handlers (map overview-only-active-handlers)
+     query (filter-with-search query)
+     true (sort-by last-activity >)
+     limit (take limit))))
+
 (def my-applications-api
   (context "/my-applications" []
     :tags ["applications"]
@@ -176,15 +184,14 @@
     (GET "/handled" []
       :summary "Get all applications that the current user no more needs to act on."
       :roles #{:logged-in}
+      ;; XXX: checking this schema can be mighty slow (thousands of applications to check individually)
       :return [schema/ApplicationOverview]
       :query-params [{query :- (describe s/Str "search query [documentation](https://github.com/CSCfi/rems/blob/master/docs/search.md)") nil}
                      {only-active-handlers :- (describe s/Bool "return only workflow handlers that are active making a smaller result") false}
                      {limit :- (describe s/Int "how many results to return") nil}]
-      (ok (cond->> (todos/get-handled-todos (getx-user-id))
-            only-active-handlers (map overview-only-active-handlers)
-            query (filter-with-search query)
-            true (sort-by last-activity >)
-            limit (take limit))))
+      (ok (get-handled-applications {:query query
+                                     :only-active-handlers only-active-handlers
+                                     :limit limit})))
 
     (POST "/create" request
       :summary "Create a new application"
diff --git a/src/clj/rems/db/applications.clj b/src/clj/rems/db/applications.clj
index f2098b105..0e7a437e8 100644
--- a/src/clj/rems/db/applications.clj
+++ b/src/clj/rems/db/applications.clj
@@ -384,14 +384,14 @@
         personalize-app (fn [app] ; NB: may return nil if should not see the app
                           (model/apply-user-permissions app userid))
         apps (->> app-ids
-                  (mapv cached-apps)
+                  (map cached-apps)
                   (keep personalize-app))]
     apps))
 
 (defn get-all-applications [userid]
   (->> userid
        get-all-applications-full
-       (mapv ->ApplicationOverview)))
+       (map ->ApplicationOverview)))
 
 (defn get-all-application-roles [userid]
   (-> (refresh-all-applications-cache!)
diff --git a/src/clj/rems/db/outbox.clj b/src/clj/rems/db/outbox.clj
index e281e81cf..f1ba75971 100644
--- a/src/clj/rems/db/outbox.clj
+++ b/src/clj/rems/db/outbox.clj
@@ -29,6 +29,32 @@
 (def ^:private validate-outbox-data
   (s/validator OutboxData))
 
+(defn puts!
+  "Puts multiple `datas` into outbox in one go.
+
+  One JSON is generated with an array of the individual outbox datas."
+  [datas]
+  (let [now (time/now)
+        amended (mapv (fn [data]
+                        {:outboxdata (-> data
+                                         (assoc :outbox/created now
+                                                :outbox/next-attempt now
+                                                :outbox/latest-error nil
+                                                :outbox/latest-attempt nil
+                                                :outbox/backoff initial-backoff)
+                                         validate-outbox-data)})
+                      datas)]
+    (mapv :id (db/puts-to-outbox! {:outboxdatas (json/generate-string amended)}))))
+
+(comment
+  ;; this is what the use of `puts!` should look like
+  (puts! [{:outbox/type :email
+           :outbox/email "first@example.com"
+           :outbox/deadline (time/now)}
+          {:outbox/type :email
+           :outbox/email "second@example.com"
+           :outbox/deadline (time/now)}]))
+
 (defn put! [data]
   (let [amended (assoc data
                        :outbox/created (DateTime/now)
diff --git a/src/clj/rems/email/core.clj b/src/clj/rems/email/core.clj
index 5a8fbc442..e16ed1ac1 100644
--- a/src/clj/rems/email/core.clj
+++ b/src/clj/rems/email/core.clj
@@ -30,35 +30,51 @@
       (template/event-to-emails (rems.application.model/enrich-event event users/get-user (constantly nil))
                                 (applications/get-application app-id)))))
 
-(defn- enqueue-email! [email]
-  (outbox/put! {:outbox/type :email
-                :outbox/email email
-                :outbox/deadline (-> (time/now) (.plus ^Period (:email-retry-period env)))}))
-
 (defn generate-event-emails! [new-events]
-  (doseq [event new-events
-          email (event-to-emails event)]
-    (enqueue-email! email)))
+  (let [deadline (-> (time/now) (.plus ^Period (:email-retry-period env)))]
+    (->> (for [event new-events
+               email (event-to-emails event)]
+           {:outbox/type :email
+            :outbox/email email
+            :outbox/deadline deadline})
+         outbox/puts!)
+    nil)) ; no new events
+
+(defn generate-handler-reminder-outbox [handler deadline]
+  (let [userid (:userid handler)
+        lang (:language (user-settings/get-user-settings userid))
+        apps (todos/get-todos userid)
+        email (template/handler-reminder-email lang handler apps)]
+
+    (when email
+      {:outbox/type :email
+       :outbox/email email
+       :outbox/deadline deadline})))
 
 (defn generate-handler-reminder-emails! []
-  (doseq [email (->> (workflow/get-handlers)
-                     (map (fn [handler]
-                            (let [lang (:language (user-settings/get-user-settings (:userid handler)))
-                                  apps (todos/get-todos (:userid handler))]
-                              (template/handler-reminder-email lang handler apps))))
-                     (remove nil?))]
-    (enqueue-email! email)))
+  (let [deadline (-> (time/now) (.plus ^Period (:email-retry-period env)))]
+    (->> (workflow/get-handlers)
+         (keep #(generate-handler-reminder-outbox % deadline))
+         outbox/puts!)))
+
+(defn generate-reviewer-reminder-outbox [reviewer deadline]
+  (let [userid (:userid reviewer)
+        lang (:language (user-settings/get-user-settings userid))
+        apps (->> (todos/get-todos userid)
+                  (filter (comp #{:waiting-for-your-review} :application/todo)))
+        email (template/reviewer-reminder-email lang reviewer apps)]
+
+    (when email
+      {:outbox/type :email
+       :outbox/email email
+       :outbox/deadline deadline})))
 
 (defn generate-reviewer-reminder-emails! []
-  (doseq [email (->> (applications/get-users-with-role :reviewer)
-                     (map users/get-user)
-                     (map (fn [reviewer]
-                            (let [lang (:language (user-settings/get-user-settings (:userid reviewer)))
-                                  apps (->> (todos/get-todos (:userid reviewer))
-                                            (map #(= :waiting-for-your-review (:application/todo %))))]
-                              (template/reviewer-reminder-email lang reviewer apps))))
-                     (remove nil?))]
-    (enqueue-email! email)))
+  (let [deadline (-> (time/now) (.plus ^Period (:email-retry-period env)))]
+    (->> (applications/get-users-with-role :reviewer)
+         (map users/get-user)
+         (keep #(generate-reviewer-reminder-outbox % deadline))
+         outbox/puts!)))
 
 (defn- render-invitation-template [invitation]
   (let [lang (:default-language env)] ; we don't know the preferred languages here since there is no user
@@ -67,12 +83,20 @@
             (assert workflow "Can't send invitation, missing workflow")
             (template/workflow-handler-invitation-email lang invitation workflow)))))
 
+(defn- mark-invitations-sent! [invitations]
+  (doseq [invitation invitations]
+    (invitation/mail-sent! (:invitation/id invitation))))
+
 (defn generate-invitation-emails! [invitations]
-  (doseq [invitation invitations
-          :when (not (:invitation/sent invitation))
-          :let [email (render-invitation-template invitation)]]
-    (invitation/mail-sent! (:invitation/id invitation))
-    (enqueue-email! email)))
+  (let [deadline (-> (time/now) (.plus ^Period (:email-retry-period env)))]
+    (->> (for [invitation invitations
+               :when (not (:invitation/sent invitation))
+               :let [email (render-invitation-template invitation)]]
+           {:outbox/type :email
+            :outbox/email email
+            :outbox/deadline deadline})
+         outbox/puts!)
+    (mark-invitations-sent! invitations)))
 
 ;;; Email poller
 
diff --git a/src/clj/rems/permissions.clj b/src/clj/rems/permissions.clj
index b75fd19f2..e6cec2d9f 100644
--- a/src/clj/rems/permissions.clj
+++ b/src/clj/rems/permissions.clj
@@ -28,10 +28,9 @@
       (update :application/user-roles dissoc-if-empty user)))
 
 (defn user-roles [application user]
-  (let [specific-roles (set (get-in application [:application/user-roles user]))]
-    (if (seq specific-roles)
-      specific-roles
-      #{:everyone-else})))
+  (if-some [specific-roles (seq (-> application :application/user-roles (get user)))]
+    (set specific-roles)
+    #{:everyone-else}))
 
 (deftest test-user-roles
   (testing "give first role"
@@ -197,10 +196,11 @@
    Union of all role specific permissions. Returns an empty set if no
    permissions are set for the user."
   [application user]
-  (->> (user-roles application user)
-       (mapcat (fn [role]
-                 (get-in application [:application/role-permissions role])))
-       set))
+  (reduce (fn [permissions role]
+            (into permissions
+                  (-> application :application/role-permissions role)))
+          #{}
+          (user-roles application user)))
 
 (deftest test-user-permissions
   (testing "unknown user"


Test patch:
diff --git a/src/clj/rems/service/test_data.clj b/src/clj/rems/service/test_data.clj
index 1401f64bc..a9ffc5f81 100644
--- a/src/clj/rems/service/test_data.clj
+++ b/src/clj/rems/service/test_data.clj
@@ -602,7 +602,8 @@
 
 (defn create-performance-test-data! []
   (log/info "Creating performance test data")
-  (let [resource-count 1000
+  (let [start-time (System/currentTimeMillis)
+        resource-count 1000
         application-count 3000
         user-count 1000
         handler-count 100
@@ -700,13 +701,15 @@
       (in-parallel
        (for [n (range-1 application-count)]
          (fn []
-           (log/info "Creating performance test application" n "/" application-count)
            (let [cat-item-id (rand-nth cat-item-ids)
                  user-id (rand-nth user-ids)
                  handler (rand-nth handlers)
                  app-id (test-helpers/create-application! {:catalogue-item-ids [cat-item-id]
                                                            :actor user-id})
                  long-answer (random-long-string)]
+
+             (log/info "Created performance test application" app-id (str "(" n "/" application-count ")"))
+
              (dotimes [i 20] ; user saves ~ 20 times while writing an application
                (test-helpers/command! {:type :application.command/save-draft
                                        :application-id app-id
@@ -730,7 +733,10 @@
                                      :application-id app-id
                                      :actor handler
                                      :comment ""}))))))
-    (log/info "Performance test applications created")))
+    (let [delta-time (/ (- (System/currentTimeMillis) start-time) 1000.0)]
+      (log/infof "Performance test applications created in %.2fs (%.2f/s)"
+                 delta-time
+                 (/ application-count delta-time)))))
 
 (defn- create-items! [users users-data]
   (let [owner (users :owner)
diff --git a/test/clj/rems/api/test_applications.clj b/test/clj/rems/api/test_applications.clj
index baf14df4e..bb156b979 100644
--- a/test/clj/rems/api/test_applications.clj
+++ b/test/clj/rems/api/test_applications.clj
@@ -3258,3 +3258,14 @@
                  (->> (get-application-for-user application-id userid)
                       :application/events
                       (mapv #(select-keys % [:event/actor :event/type :event/public :application/processing-state]))))))))))
+
+
+(comment
+  ;; This is a way to test the whole API performance
+  ;; e.g. Schema checking and JSON serialization in addition
+  ;; to the business logic.
+  ;;
+  ;; An alternative would be to do the same as the fixtures do
+  ;; to mount a proper handler.
+  (deftest test-todos-performance
+    (clj-async-profiler.core/profile {:interval 100000} (get-handled-todos "handler"))))
diff --git a/test/clj/rems/api/test_email.clj b/test/clj/rems/api/test_email.clj
index d9c9ca590..ff75d106c 100644
--- a/test/clj/rems/api/test_email.clj
+++ b/test/clj/rems/api/test_email.clj
@@ -37,8 +37,8 @@
   (create-application-in-progress!)
   (testing "sends emails"
     (let [outbox-emails (atom [])]
-      (with-redefs [outbox/put! (fn [email]
-                                  (swap! outbox-emails conj email))]
+      (with-redefs [outbox/puts! (fn [emails]
+                                   (swap! outbox-emails concat emails))]
         (let [body (-> (request :post "/api/email/send-handler-reminder")
                        (authenticate test-data/+test-api-key+ "developer")
                        handler
@@ -64,8 +64,8 @@
 
   (testing "sends emails"
     (let [outbox-emails (atom [])]
-      (with-redefs [outbox/put! (fn [email]
-                                  (swap! outbox-emails conj email))]
+      (with-redefs [outbox/puts! (fn [emails]
+                                   (swap! outbox-emails concat emails))]
         (let [body (-> (request :post "/api/email/send-reviewer-reminder")
                        (authenticate test-data/+test-api-key+ "developer")
                        handler
@@ -91,8 +91,8 @@
 
   (testing "sends emails"
     (let [outbox-emails (atom [])]
-      (with-redefs [outbox/put! (fn [email]
-                                  (swap! outbox-emails conj email))]
+      (with-redefs [outbox/puts! (fn [emails]
+                                   (swap! outbox-emails concat emails))]
         (let [body (-> (request :post "/api/email/send-reminders")
                        (authenticate test-data/+test-api-key+ "developer")
                        handler
diff --git a/test/clj/rems/application/test_eraser.clj b/test/clj/rems/application/test_eraser.clj
index 4349c28ab..8513b4d13 100644
--- a/test/clj/rems/application/test_eraser.clj
+++ b/test/clj/rems/application/test_eraser.clj
@@ -73,7 +73,7 @@
         outbox-emails (atom [])]
 
     (testing "does not process applications when expirer-bot user does not exist"
-      (with-redefs [outbox/put! (fn [email] (swap! outbox-emails conj email))
+      (with-redefs [outbox/puts! (fn [emails] (swap! outbox-emails concat emails))
                     env {:application-expiration {:application.state/draft {:delete-after "P90D"}}}]
         (log-test/with-log
           (testing "processing applications does not delete applications"
@@ -92,7 +92,7 @@
           (is (log-test/logged? "rems.application.eraser" :warn "Cannot process applications, because user expirer-bot does not exist")))))
 
     (testing "does not delete applications when configuration is not valid"
-      (with-redefs [outbox/put! (fn [email] (swap! outbox-emails conj email))
+      (with-redefs [outbox/puts! (fn [emails] (swap! outbox-emails concat emails))
                     env {}]
         (test-helpers/create-user! {:userid expirer-bot/bot-userid})
         (roles/add-role! expirer-bot/bot-userid :expirer)
@@ -116,7 +116,7 @@
           (is (log-test/logged? "rems.application.eraser" :info "No applications to process")))))
 
     (testing "cannot remove other than draft applications"
-      (with-redefs [outbox/put! (fn [email] (swap! outbox-emails conj email))
+      (with-redefs [outbox/puts! (fn [emails] (swap! outbox-emails concat emails))
                     env {:application-expiration {:application.state/submitted {:delete-after "P90D"}}}]
         (log-test/with-log
           (testing "processing applications does not delete applications"
@@ -146,7 +146,7 @@
             (is (not (log-test/logged? "rems.db.applications" :info #"Finished deleting application")))))))
 
     (testing "deletes expired draft application"
-      (with-redefs [outbox/put! (fn [email] (swap! outbox-emails conj email))
+      (with-redefs [outbox/puts! (fn [emails] (swap! outbox-emails concat emails))
                     env {:application-expiration {:application.state/draft {:delete-after "P90D"}}}]
         (with-fixed-time test-time
           (log-test/with-log
@@ -198,7 +198,7 @@
       (doseq [expiration [{:application.state/draft {:delete-after "P90D"}}
                           {:application.state/draft {:reminder-before "P7D"}}
                           nil]]
-        (with-redefs [outbox/put! (fn [email] (swap! outbox-emails conj email))
+        (with-redefs [outbox/puts! (fn [emails] (swap! outbox-emails concat emails))
                       env {:application-id-column :id
                            :public-url "localhost/"
                            :application-expiration expiration}]
@@ -218,7 +218,7 @@
                 (is (empty? @outbox-emails))))))))
 
     (testing "send expiration notifications"
-      (with-redefs [outbox/put! (fn [email] (swap! outbox-emails conj email))
+      (with-redefs [outbox/puts! (fn [emails] (swap! outbox-emails concat emails))
                     user-settings/get-user-settings (constantly {:language :en})
                     env {:application-id-column :id
                          :public-url "localhost/"
diff --git a/test/clj/rems/service/test_invitation.clj b/test/clj/rems/service/test_invitation.clj
index 37875b686..23fc9af89 100644
--- a/test/clj/rems/service/test_invitation.clj
+++ b/test/clj/rems/service/test_invitation.clj
@@ -3,7 +3,7 @@
             [rems.service.invitation :as invitation]
             [rems.db.test-data-helpers :as test-helpers]
             [rems.db.testing :refer [reset-caches-fixture rollback-db-fixture test-db-fixture]]
-            [rems.email.core :as email]
+            [rems.db.outbox :as outbox]
             [rems.testing-util :refer [fixed-time-fixture with-user]])
   (:import [org.joda.time DateTime DateTimeUtils DateTimeZone]))
 
@@ -44,8 +44,8 @@
                                                  :workflow-id 100042}))))
 
         (testing "success"
-          (let [sent-email (atom nil)]
-            (with-redefs [email/enqueue-email! (fn [& args] (reset! sent-email args))]
+          (let [sent-emails (atom nil)]
+            (with-redefs [outbox/puts! (fn [emails] (reset! sent-emails emails))]
               (let [invitation (invitation/create-invitation! {:userid "owner"
                                                                :name "Dorothy Vaughan"
                                                                :email "dorothy.vaughan@nasa.gov"
@@ -53,7 +53,7 @@
                     invitation-id (:invitation/id invitation)]
                 (is (= {:success true} (dissoc invitation :invitation/id)))
                 (is (number? invitation-id))
-                (is (= "dorothy.vaughan@nasa.gov" (:to (first @sent-email))))
+                (is (= "dorothy.vaughan@nasa.gov" (:to (:outbox/email (first @sent-emails)))))
 
                 (testing "after creating invitations"
                   (let [invitations (invitation/get-invitations nil)]