Repository: CSCfi/rems
Pull request #1106 (2019-04-10 13:05:53)

Performance test data
#863

The most critical performance optimizations have been done. More will follow.


Patch:
diff --git a/src/clj/rems/api/applications_v2.clj b/src/clj/rems/api/applications_v2.clj
index 815755527..a9b2091c9 100644
--- a/src/clj/rems/api/applications_v2.clj
+++ b/src/clj/rems/api/applications_v2.clj
@@ -1,5 +1,8 @@
 (ns rems.api.applications-v2
-  (:require [rems.application.model :as model]
+  (:require [clojure.tools.logging :as log]
+            [medley.core :refer [map-vals]]
+            [mount.core :as mount]
+            [rems.application.model :as model]
             [rems.auth.util :refer [throw-forbidden]]
             [rems.db.applications :as applications]
             [rems.db.catalogue :as catalogue]
@@ -65,15 +68,26 @@
           :application/form
           :application/licenses))
 
+(mount/defstate all-applications-cache
+  :start (atom {:last-processed-event-id 0
+                :applications nil}))
+
 (defn get-all-applications [user-id]
-  ;; TODO: cache the applications and build the projection incrementally as new events are published
-  (let [events (applications/get-dynamic-application-events-since 0)
-        applications (reduce all-applications-view nil events)]
+  (let [cache @all-applications-cache
+        events (applications/get-dynamic-application-events-since (:last-processed-event-id cache))
+        applications (reduce all-applications-view (:applications cache) events)
+        ;; TODO: for a shared cache it may be necessary to make assoc-injections idempotent and consider cache invalidation
+        cached-injections (map-vals memoize injections)]
+    (when-let [event-id (:event/id (last events))]
+      (when (compare-and-set! all-applications-cache
+                              cache
+                              {:last-processed-event-id event-id
+                               :applications applications})
+        (log/info "Updated all-applications-cache from" (:last-processed-event-id cache) "to" event-id)))
     (->> (vals applications)
          (map #(model/apply-user-permissions % user-id))
          (remove nil?)
-         ;; TODO: for caching it may be necessary to make assoc-injections idempotent and consider cache invalidation
-         (map #(model/enrich-with-injections % injections))
+         (map #(model/enrich-with-injections % cached-injections))
          (map exclude-unnecessary-keys-from-overview))))
 
 (defn- own-application? [application]
diff --git a/src/clj/rems/auth/fake_shibboleth.clj b/src/clj/rems/auth/fake_shibboleth.clj
index 13d4663a7..c4929024a 100644
--- a/src/clj/rems/auth/fake_shibboleth.clj
+++ b/src/clj/rems/auth/fake_shibboleth.clj
@@ -1,12 +1,12 @@
 (ns rems.auth.fake-shibboleth
-  (:require [compojure.core :refer [GET defroutes]]
+  (:require [clojure.string :as str]
+            [compojure.core :refer [GET defroutes]]
             [hiccup.page :refer [html5]]
             [hiccup.util :refer [url]]
             [rems.db.core :as db]
             [rems.db.test-data :refer [+fake-user-data+]]
             [rems.json :as json]
-            [ring.util.response :refer [content-type redirect
-                                        response]]))
+            [ring.util.response :refer [content-type redirect response]]))
 
 (def ^{:private true
        :doc "Inlined CSS declaration for fake login."}
@@ -63,6 +63,7 @@ a:visited { color: #fff; }
                 [:div.login
                  [:h1 "Development Login"]
                  [:div.users (->> (map :userid (db/get-users))
+                                  (remove #(str/starts-with? % "perftester"))
                                   (sort)
                                   (distinct)
                                   (map user-selection))]]])
diff --git a/src/clj/rems/db/workflow.clj b/src/clj/rems/db/workflow.clj
index 21c112b36..b6e4c57fc 100644
--- a/src/clj/rems/db/workflow.clj
+++ b/src/clj/rems/db/workflow.clj
@@ -34,6 +34,9 @@
 
 (defn- create-dynamic-workflow! [{:keys [user-id organization title handlers]}]
   (assert user-id)
+  (assert organization)
+  (assert title)
+  (assert (every? string? handlers) {:handlers handlers})
   (let [wfid (:id (db/create-workflow! {:organization organization,
                                         :owneruserid user-id,
                                         :modifieruserid user-id,
diff --git a/src/clj/rems/standalone.clj b/src/clj/rems/standalone.clj
index 57c30b536..230dfec02 100644
--- a/src/clj/rems/standalone.clj
+++ b/src/clj/rems/standalone.clj
@@ -91,7 +91,8 @@
     (= "test-data" (first args))
     (do
       (mount/start #'rems.config/env #'rems.db.core/*db*)
-      (test-data/create-test-data!))
+      (test-data/create-test-data!)
+      (test-data/create-performance-test-data!))
     (= "demo-data" (first args))
     (do
       (mount/start #'rems.config/env #'rems.db.core/*db*)


Test patch:
diff --git a/src/clj/rems/db/test_data.clj b/src/clj/rems/db/test_data.clj
index ecf302f9c..5861b9cb6 100644
--- a/src/clj/rems/db/test_data.clj
+++ b/src/clj/rems/db/test_data.clj
@@ -6,13 +6,16 @@
             [rems.db.catalogue :as catalogue]
             [rems.db.core :as db]
             [rems.db.form :as form]
+            [rems.db.licenses :as licenses]
+            [rems.db.resource :as resource]
             [rems.db.roles :as roles]
             [rems.db.users :as users]
             [rems.db.workflow :as workflow]
             [rems.db.workflow-actors :as actors]
             [rems.locales :as locales]
             [ring.util.http-response :refer [bad-request!]])
-  (:import (org.joda.time DateTimeUtils DateTime)))
+  (:import [java.util UUID]
+           [org.joda.time DateTimeUtils DateTime]))
 
 (def ^DateTime creation-time (time/now)) ; TODO: no more used, remove?
 
@@ -628,6 +631,92 @@
       (let [application (create-draft! applicant item-without-new-license wfid "applied before license was valid" (time/minus (time/now) (time/days 2)))]
         (applications/submit-application applicant application)))))
 
+(defn create-performance-test-data! []
+  (let [resource-count 1000
+        application-count 1000
+        user-count 1000
+        handlers [(+fake-users+ :approver1)
+                  (+fake-users+ :approver2)]
+        owner (+fake-users+ :owner)
+        workflow-id (:id (workflow/create-workflow! {:user-id owner
+                                                     :organization "perf"
+                                                     :title "Performance tests"
+                                                     :type :dynamic
+                                                     :handlers handlers}))
+        form-id (:id (form/create-form!
+                      owner
+                      {:organization "perf"
+                       :title "Performance tests"
+                       :items [{:title {:en "Project name"
+                                        :fi "Projektin nimi"}
+                                :optional false
+                                :type "description"
+                                :input-prompt {:en "Project"
+                                               :fi "Projekti"}}
+
+                               {:title {:en "Project description"
+                                        :fi "Projektin kuvaus"}
+                                :optional false
+                                :type "texta"
+                                :input-prompt {:en "The purpose of the project is to..."
+                                               :fi "Projektin tarkoitus on..."}}]}))
+        form (form/get-form form-id)
+        license-id (:id (licenses/create-license!
+                         {:licensetype "text"
+                          :title "Performance License"
+                          :textcontent "Be fast."
+                          :localizations {:en {:title "Performance license"
+                                               :textcontent "Be fast."}
+                                          :fi {:title "Suorituskykylisenssi"
+                                               :textcontent "Ole nopea."}}}
+                         owner))
+        cat-item-ids (vec (for [_ (range resource-count)]
+                            (let [uuid (UUID/randomUUID)
+                                  resource (resource/create-resource!
+                                            {:resid (str "urn:uuid:" uuid)
+                                             :organization "perf"
+                                             :licenses [license-id]}
+                                            owner)
+                                  _ (assert (:success resource))
+                                  cat-item (catalogue/create-catalogue-item! {:title (str "Performance test resource " uuid)
+                                                                              :form form-id
+                                                                              :resid (:id resource)
+                                                                              :wfid workflow-id})]
+                              (:id cat-item))))
+        user-ids (vec (for [n (range 1 (inc user-count))]
+                        (let [user-id (str "perftester" n)]
+                          (users/add-user! user-id {:eppn user-id
+                                                    :mail (str user-id "@example.com")
+                                                    :commonName (str "Performance Tester " n)})
+                          user-id)))]
+    (dotimes [_ application-count]
+      (let [cat-item-id (rand-nth cat-item-ids)
+            user-id (rand-nth user-ids)
+            handler (rand-nth handlers)
+            app-id (:application-id (applications/create-application! user-id [cat-item-id]))]
+        (assert (nil? (applications/command!
+                       {:type :application.command/save-draft
+                        :actor user-id
+                        :time (time/now)
+                        :application-id app-id
+                        :field-values [{:field (:id (first (:fields form)))
+                                        :value (str "Performance test application " (UUID/randomUUID))}
+                                       {:field (:id (second (:fields form)))
+                                        ;; 5000 characters (10 KB) of lorem ipsum generated with www.lipsum.com
+                                        ;; to increase the memory requirements of an application
+                                        :value "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut non diam vel erat dapibus facilisis vel vitae nunc. Curabitur at fermentum lorem. Cras et bibendum ante. Etiam convallis erat justo. Phasellus cursus molestie vehicula. Etiam molestie tellus vitae consectetur dignissim. Pellentesque euismod hendrerit mi sed tincidunt. Integer quis lorem ut ipsum egestas hendrerit. Aenean est nunc, mattis euismod erat in, sodales rutrum mauris. Praesent sit amet risus quis felis congue ultricies. Nulla facilisi. Sed mollis justo id tristique volutpat.\n\nPhasellus augue mi, facilisis ac velit et, pharetra tristique nunc. Pellentesque eget arcu quam. Curabitur dictum nulla varius hendrerit varius. Proin vulputate, ex lacinia commodo varius, ipsum velit viverra est, eget molestie dui nisi non eros. Nulla lobortis odio a magna mollis placerat. Interdum et malesuada fames ac ante ipsum primis in faucibus. Integer consectetur libero ut gravida ullamcorper. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Donec aliquam feugiat mollis. Quisque massa lacus, efficitur vel justo vel, elementum mollis magna. Maecenas at sem sem. Praesent sed ex mattis, egestas dui non, volutpat lorem. Nulla tempor, nisi rutrum accumsan varius, tellus elit faucibus nulla, vel mattis lacus justo at ante. Sed ut mollis ex, sed tincidunt ex.\n\nMauris laoreet nibh eget erat tincidunt pharetra. Aenean sagittis maximus consectetur. Curabitur interdum nibh sed tincidunt finibus. Sed blandit nec lorem at iaculis. Morbi non augue nec tortor hendrerit mollis ut non arcu. Suspendisse maximus nec ligula a efficitur. Etiam ultrices rhoncus leo quis dapibus. Integer vel rhoncus est. Integer blandit varius auctor. Vestibulum suscipit suscipit risus, sit amet venenatis lacus iaculis a. Duis eu turpis sit amet nibh sagittis convallis at quis ligula. Sed eget justo quis risus iaculis lacinia vitae a justo. In hac habitasse platea dictumst. Maecenas euismod et lorem vel viverra.\n\nDonec bibendum nec ipsum in volutpat. Vivamus in elit venenatis, venenatis libero ac, ultrices dolor. Morbi quis odio in neque consequat rutrum. Suspendisse quis sapien id sapien fermentum dignissim. Nam eu est vel risus volutpat mollis sed quis eros. Proin leo nulla, dictum id hendrerit vitae, scelerisque in elit. Proin consectetur sodales arcu ac tristique. Suspendisse ut elementum ligula, at rhoncus mauris. Aliquam lacinia at diam eget mattis. Phasellus quam leo, hendrerit sit amet mi eget, porttitor aliquet velit. Proin turpis ante, consequat in enim nec, tempus consequat magna. Vestibulum fringilla ac turpis nec malesuada. Proin id lectus iaculis, suscipit erat at, volutpat turpis. In quis faucibus elit, ut maximus nibh. Sed egestas egestas dolor.\n\nNulla varius orci quam, id auctor enim ultrices nec. Morbi et tellus ac metus sodales convallis sed vehicula neque. Pellentesque rhoncus mattis massa a bibendum. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Fusce tincidunt nulla non aliquet facilisis. Praesent nisl nisi, finibus id odio sed, consectetur feugiat mauris. Suspendisse sed lacinia ligula. Duis vitae nisl leo. Donec erat arcu, feugiat sit amet sagittis ac, scelerisque nec est. Pellentesque finibus mauris nulla, in maximus sapien pharetra vitae. Sed leo elit, consequat eu aliquam vitae, feugiat ut eros. Pellentesque dictum feugiat odio sed commodo. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin neque quam, varius vel libero sit amet, rhoncus sollicitudin ex. In a dui non neque malesuada pellentesque.\n\nProin tincidunt nisl non commodo faucibus. Sed porttitor arcu neque, vitae bibendum sapien placerat nec. Integer eget tristique orci. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Donec eu molestie eros. Nunc iaculis rhoncus enim, vel mattis felis fringilla condimentum. Interdum et malesuada fames ac ante ipsum primis in faucibus. Aenean ac augue nulla. Phasellus vitae nulla lobortis, mattis magna ac, gravida ipsum. Aenean ornare non nunc non luctus. Aenean lacinia lectus nec velit finibus egestas vel ut ipsum. Cras hendrerit rhoncus erat, vel maximus nunc.\n\nPraesent quis imperdiet quam. Praesent ligula tellus, consectetur sed lacus eu, malesuada condimentum tellus. Donec et diam hendrerit, dictum diam quis, aliquet purus. Suspendisse pulvinar neque at efficitur iaculis. Nulla erat orci, euismod id velit sed, dictum hendrerit arcu. Nulla aliquam molestie aliquam. Duis et semper nisi, eget commodo arcu. Praesent rhoncus, nulla id sodales eleifend, ante ipsum pellentesque augue, id iaculis sem est vitae est. Phasellus cursus diam a lorem vestibulum sodales. Nullam lacinia tortor vel tellus commodo, sit amet sodales quam malesuada.\n\nNulla tempor lectus vel arcu feugiat, vel dapibus ex dapibus. Maecenas purus justo, aliquet et sem sit amet, tincidunt venenatis dui. Nulla eget purus id sapien elementum rutrum eu vel libero. Cras non accumsan justo posuere."}]})))
+        (assert (nil? (applications/command!
+                       {:type :application.command/submit
+                        :actor user-id
+                        :time (time/now)
+                        :application-id app-id})))
+        (assert (nil? (applications/command!
+                       {:type :application.command/approve
+                        :actor handler
+                        :time (time/now)
+                        :application-id app-id
+                        :comment "Looks fine."})))))))
+
 (defn create-test-data! []
   (DateTimeUtils/setCurrentMillisFixed (.getMillis creation-time))
   (try
diff --git a/test/clj/rems/api/test_entitlements.clj b/test/clj/rems/api/test_entitlements.clj
index bba4004fd..fb2b22b0d 100644
--- a/test/clj/rems/api/test_entitlements.clj
+++ b/test/clj/rems/api/test_entitlements.clj
@@ -1,8 +1,9 @@
 (ns ^:integration rems.api.test-entitlements
   (:require [clojure.test :refer :all]
-            [rems.handler :refer [handler]]
             [rems.api.testing :refer :all]
             [rems.db.users :as users]
+            [rems.handler :refer [handler]]
+            [rems.poller.entitlements :as entitlements-poller]
             [ring.mock.request :refer :all]))
 
 (use-fixtures
@@ -17,7 +18,7 @@
       (is (= "unauthorized" body)))))
 
 (deftest entitlements-test
-  (rems.poller.entitlements/run)
+  (entitlements-poller/run)
   (let [api-key "42"]
     (testing "all"
       (let [data (-> (request :get "/api/entitlements")
diff --git a/test/clj/rems/db/test_entitlements.clj b/test/clj/rems/db/test_entitlements.clj
index b2ef23f6a..669e1e04b 100644
--- a/test/clj/rems/db/test_entitlements.clj
+++ b/test/clj/rems/db/test_entitlements.clj
@@ -2,12 +2,13 @@
   (:require [cheshire.core :as cheshire]
             [clj-time.core :as time]
             [clojure.test :refer :all]
-            [rems.db.core :as db]
             [rems.db.applications :as applications]
+            [rems.db.core :as db]
             [rems.db.entitlements :as entitlements]
             [rems.db.licenses :as licenses]
             [rems.db.resource :as resource]
             [rems.db.testing :refer [test-db-fixture rollback-db-fixture test-data-fixture]]
+            [rems.poller.entitlements :as entitlements-poller]
             [rems.testing-util :refer [suppress-logging]]
             [stub-http.core :as stub]))
 
@@ -69,7 +70,7 @@
             (is (= +expected-payload+ (cheshire/parse-string payload)))))))))
 
 (deftest test-entitlement-granting
-  (rems.poller.entitlements/run) ; clear previous entitlements
+  (entitlements-poller/run) ; clear previous entitlements
   (with-open [server (stub/start! {"/add" {:status 200}
                                    "/remove" {:status 200}})]
     (with-redefs [rems.config/env {:entitlements-target
@@ -110,7 +111,7 @@
                                             :member {:userid memberid}
                                             :time (time/now)})))
           (testing "submitted application should not yet cause entitlements"
-            (rems.poller.entitlements/run)
+            (entitlements-poller/run)
             (is (empty? (db/get-entitlements {:application app-id})))
             (is (empty? (stub/recorded-requests server))))
 
@@ -121,7 +122,7 @@
                                             :time (time/now)})))
 
           (testing "approved application should not yet cause entitlements"
-            (rems.poller.entitlements/run)
+            (entitlements-poller/run)
             (is (empty? (db/get-entitlements {:application app-id})))
             (is (empty? (stub/recorded-requests server))))
 
@@ -142,8 +143,8 @@
                  (:application/accepted-licenses (applications/get-application-state app-id))))
 
           (testing "approved application, licenses accepted by one user generates entitlements for that user"
-            (rems.poller.entitlements/run)
-            (rems.poller.entitlements/run) ;; run twice to check idempotence
+            (entitlements-poller/run)
+            (entitlements-poller/run) ;; run twice to check idempotence
             (is (= 2 (count (stub/recorded-requests server))))
             (testing "db"
               (is (= [[uid "resource1"] [uid "resource2"]]
@@ -167,7 +168,7 @@
                                             :time (time/now)})))
 
           (testing "approved application, more accepted licenses generates more entitlements"
-            (rems.poller.entitlements/run)
+            (entitlements-poller/run)
             (is (= 4 (count (stub/recorded-requests server))))
             (testing "db"
               (is (= [[uid "resource1"] [uid "resource2"]
@@ -193,7 +194,7 @@
                                             :time (time/now)})))
 
           (testing "removing a member ends entitlements"
-            (rems.poller.entitlements/run)
+            (entitlements-poller/run)
             (is (= 5 (count (stub/recorded-requests server))))
             (testing "db"
               (is (= [[uid "resource1"] [uid "resource2"]]
@@ -214,7 +215,7 @@
                                             :time (time/now)})))
 
           (testing "closed application should end entitlements"
-            (rems.poller.entitlements/run)
+            (entitlements-poller/run)
             (is (= 6 (count (stub/recorded-requests server))))
             (testing "db"
               (is (= []
diff --git a/test/clj/rems/db/testing.clj b/test/clj/rems/db/testing.clj
index 33032e00d..33dccb91b 100644
--- a/test/clj/rems/db/testing.clj
+++ b/test/clj/rems/db/testing.clj
@@ -11,7 +11,8 @@
   (mount/stop) ;; during interactive development, app might be running when tests start. we need to tear it down
   (mount/start-with-args {:test true}
                          #'rems.config/env
-                         #'rems.db.core/*db*)
+                         #'rems.db.core/*db*
+                         #'rems.api.applications-v2/all-applications-cache)
   (db/assert-test-database!)
   (migrations/migrate ["reset"] {:database-url (:test-database-url env)})
   (f)
diff --git a/test/clj/rems/test_browser.clj b/test/clj/rems/test_browser.clj
index 9c4d8cd97..3a96bd0dd 100644
--- a/test/clj/rems/test_browser.clj
+++ b/test/clj/rems/test_browser.clj
@@ -40,6 +40,7 @@
   (mount/start)
   (migrations/migrate ["reset"] (select-keys rems.config/env [:database-url]))
   (test-data/create-test-data!)
+  (test-data/create-performance-test-data!)
   (f)
   (mount/stop))
 
diff --git a/test/clj/rems/test_db.clj b/test/clj/rems/test_db.clj
index 30ab2fae4..3977290b8 100644
--- a/test/clj/rems/test_db.clj
+++ b/test/clj/rems/test_db.clj
@@ -17,10 +17,9 @@
             [rems.db.testing :refer [test-db-fixture rollback-db-fixture]]
             [rems.db.users :as users]
             [rems.db.workflow-actors :as actors]
-            [rems.poller.entitlements]
+            [rems.poller.entitlements :as entitlements-poller]
             [rems.testing-tempura :refer [fake-tempura-fixture]]
-            [rems.util :refer [get-user-id]]
-            [stub-http.core :as stub])
+            [rems.util :refer [get-user-id]])
   (:import (rems.auth ForbiddenException)))
 
 (use-fixtures :once fake-tempura-fixture test-db-fixture)
@@ -200,7 +199,7 @@
                                       :comment ""})))
     (is (= :application.state/approved (:application/state (applications-v2/get-application uid app-id))))
 
-    (rems.poller.entitlements/run)
+    (entitlements-poller/run)
     (is (= ["resid111" "resid222"] (sort (map :resid (db/get-entitlements {:application app-id}))))
         "should create entitlements for both resources")))
 
@@ -648,7 +647,7 @@
                                           :actor "handler"
                                           :application-id jill-app
                                           :comment ""})))
-    (rems.poller.entitlements/run)
+    (entitlements-poller/run)
 
     (binding [context/*roles* #{:handler}]
       (let [lines (split-lines (entitlements/get-entitlements-for-export))]