Repository: accelerated-text/accelerated-text (language: python)

Issue:
Async parameter was removed from text generation endpoint
There was this nice feature added in PR https://github.com/tokenmill/accelerated-text/pull/152, which seems to be removed now.

It could ease the use of this endpoint in various cases.


Discussion / hints:

Fix patch:
diff --git a/api/src/api/config.clj b/api/src/api/config.clj
index adbe27086..c0e8d6998 100644
--- a/api/src/api/config.clj
+++ b/api/src/api/config.clj
@@ -7,16 +7,17 @@
   (map (comp str/capitalize str/trim) (str/split l #",")))
 
 (defn load-config []
-  {:port               (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001"))
-   :host               (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0")
-   :db-implementation  (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")]
-                         (keyword db-implementation))
-   :db-uri             (System/getenv "DB_URI")
-   :enabled-languages  (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng")))
-   :enabled-readers    (set (read-conf-line (or (System/getenv "ENABLED_READERS") "")))
-   :config-path        (or (System/getenv "CONFIG_PATH") (io/resource "config"))
-   :dictionary-path    (or (System/getenv "DICT_PATH") (io/resource "dictionary"))
-   :document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans"))
+  {:port                 (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001"))
+   :host                 (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0")
+   :db-implementation    (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")]
+                           (keyword db-implementation))
+   :db-uri               (System/getenv "DB_URI")
+   :display-error        (Boolean/valueOf (System/getenv "DISPLAY_ERROR"))
+   :enabled-languages    (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng")))
+   :enabled-readers      (set (read-conf-line (or (System/getenv "ENABLED_READERS") "")))
+   :config-path          (or (System/getenv "CONFIG_PATH") (io/resource "config"))
+   :dictionary-path      (or (System/getenv "DICT_PATH") (io/resource "dictionary"))
+   :document-plan-path   (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans"))
    :relevant-items-limit (or (System/getenv "RELEVANT_ITEMS_MATRIX_LIMIT") 100)})
 
 (defstate conf :start (load-config))
diff --git a/api/src/api/nlg/service.clj b/api/src/api/nlg/service.clj
index fa6fd1f65..960abbc81 100644
--- a/api/src/api/nlg/service.clj
+++ b/api/src/api/nlg/service.clj
@@ -1,7 +1,7 @@
 (ns api.nlg.service
   (:require [api.nlg.core :refer [generate-text]]
-            [api.nlg.format :refer [use-format with-default-format]]
             [api.nlg.service.request :as request]
+            [api.nlg.service.response :as response]
             [api.nlg.service.utils :as utils]
             [api.utils :refer [gen-uuid]]
             [clojure.spec.alpha :as s]
@@ -16,7 +16,8 @@
                    ::request/dataId
                    ::request/dataRow
                    ::request/sampleMethod
-                   ::request/readerFlagValues]))
+                   ::request/readerFlagValues
+                   ::request/async]))
 
 (s/def ::generate-request-bulk
   (s/keys :req-un [::request/dataRows]
@@ -24,26 +25,38 @@
                    ::request/documentPlanName
                    ::request/readerFlagValues]))
 
+(s/def ::generate-response
+  (s/keys :req-un [::response/resultId]
+          :opt-un [::response/offset
+                   ::response/totalCount
+                   ::response/ready
+                   ::response/updatedAt
+                   ::response/variants]))
+
 (s/def ::get-result
   (s/keys :opt-un [::request/format]))
 
-(defn display-error? []
-  (Boolean/valueOf (System/getenv "DISPLAY_ERROR")))
-
 (defn generate-request
-  [{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues :as request}]
+  [{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues async :async :as request :or {async true}}]
   (try
     (log/infof "Generate request with %s" (utils/request->text request))
     (let [{row-index :dataSampleRow :as document-plan} (utils/get-document-plan request)
-          result-id (gen-uuid)]
+          result-id (gen-uuid)
+          body {:id            result-id
+                :document-plan document-plan
+                :data          (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {})
+                :reader-model  (map reader-model/update! (utils/form-reader-model reader-model))}]
       (results/write #::result{:id     result-id
                                :status :pending})
-      (results/write (generate-text {:id            result-id
-                                     :document-plan document-plan
-                                     :data          (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {})
-                                     :reader-model  (map reader-model/update! (utils/form-reader-model reader-model))}))
-      {:status 200
-       :body   {:resultId result-id}})
+      (if (true? async)
+        (do
+          (future (results/write (generate-text body)))
+          {:status 200
+           :body   {:resultId result-id}})
+        (do
+          (results/write (generate-text body))
+          {:status 200
+           :body   (utils/translate-result (results/fetch result-id) {:format "raw"})})))
     (catch Exception e
       (utils/error-response e "Generate request failure"))))
 
@@ -73,16 +86,9 @@
 (defn get-result [{{{request-id :id} :path {result-format :format} :query} :parameters}]
   (try
     (log/infof "Result request with id `%s`" request-id)
-    (if-let [{::result/keys [rows status timestamp] :as result} (results/fetch request-id)]
+    (if-let [result (results/fetch request-id)]
       {:status 200
-       :body   {:offset     0
-                :totalCount (count rows)
-                :ready      (not= status :pending)
-                :updatedAt  timestamp
-                :variants   (cond
-                              (some? result-format) (use-format result-format result)
-                              (and (= :error status) (display-error?)) (use-format "error" result)
-                              :else (with-default-format result))}}
+       :body   (utils/translate-result result {:format result-format})}
       (do
         (log/warnf "Result with id `%s` not found" request-id)
         {:status 404}))
diff --git a/api/src/api/nlg/service/request.clj b/api/src/api/nlg/service/request.clj
index 86e2d8a0e..3a65744df 100644
--- a/api/src/api/nlg/service/request.clj
+++ b/api/src/api/nlg/service/request.clj
@@ -16,3 +16,5 @@
 (s/def ::dataRows (s/map-of ::id ::dataRow))
 
 (s/def ::readerFlagValues (s/map-of string? boolean?))
+
+(s/def ::async boolean?)
diff --git a/api/src/api/nlg/service/response.clj b/api/src/api/nlg/service/response.clj
new file mode 100644
index 000000000..efc053184
--- /dev/null
+++ b/api/src/api/nlg/service/response.clj
@@ -0,0 +1,14 @@
+(ns api.nlg.service.response
+  (:require [clojure.spec.alpha :as s]))
+
+(s/def ::resultId string?)
+
+(s/def ::offset int?)
+
+(s/def ::totalCount int?)
+
+(s/def ::ready boolean?)
+
+(s/def ::updatedAt number?)
+
+(s/def ::variants sequential?)
diff --git a/api/src/api/nlg/service/utils.clj b/api/src/api/nlg/service/utils.clj
index e9c3f821c..5b17d829b 100644
--- a/api/src/api/nlg/service/utils.clj
+++ b/api/src/api/nlg/service/utils.clj
@@ -1,10 +1,24 @@
 (ns api.nlg.service.utils
-  (:require [api.nlg.enrich.data :as data-enrich]
+  (:require [api.config :refer [conf]]
+            [api.nlg.enrich.data :as data-enrich]
+            [api.nlg.format :refer [use-format with-default-format]]
             [clojure.string :as str]
             [clojure.tools.logging :as log]
             [data.entities.data-files :as data-files]
             [data.entities.document-plan :as dp]
-            [data.entities.reader-model :as reader-model]))
+            [data.entities.reader-model :as reader-model]
+            [data.spec.result :as result]))
+
+(defn translate-result [{::result/keys [id status timestamp rows] :as result} {result-format :format}]
+  {:resultId   id
+   :offset     0
+   :totalCount (count rows)
+   :ready      (not= status :pending)
+   :updatedAt  timestamp
+   :variants   (cond
+                 (some? result-format) (use-format result-format result)
+                 (and (= :error status) (:display-error conf)) (use-format "error" result)
+                 :else (with-default-format result))})
 
 (defn error-response
   ([exception] (error-response exception nil))
@@ -32,8 +46,8 @@
   (log/infof "Sample Method: %s" sample-method)
   (when-not (str/blank? data-id)
     (if-let [{[{fields :fields}] :records filename :fileName} (case sample-method
-                                                                    "relevant" (data-files/fetch-most-relevant data-id index 20)
-                                                                    "first"    (data-files/fetch data-id index 1))]
+                                                                "relevant" (data-files/fetch-most-relevant data-id index 20)
+                                                                "first" (data-files/fetch data-id index 1))]
       (cond->> (zipmap (map :fieldName fields) (map :value fields))
                (data-enrich/enable-enrich?) (data-enrich/enrich filename))
       (log/errorf "Data with id `%s` not found" data-id))))
diff --git a/api/src/api/server.clj b/api/src/api/server.clj
index 4f2ddeedd..3a0d2b2a5 100755
--- a/api/src/api/server.clj
+++ b/api/src/api/server.clj
@@ -34,9 +34,9 @@
   (let [main-deps {:gf (gf-service/ping)}
         other-deps {}
         color (cond
-               (some false? (vals main-deps))  :red
-               (some false? (vals other-deps)) :yellow
-               :else                           :green)]
+                (some false? (vals main-deps)) :red
+                (some false? (vals other-deps)) :yellow
+                :else :green)]
     {:status 200 :body {:color (name color) :services (merge main-deps other-deps)}}))
 
 (defn string-store [item]
@@ -62,7 +62,7 @@
                              :summary "GraphQL endpoint"}
                    :options cors-handler}]
      ["/nlg/" {:post    {:parameters {:body ::service/generate-request}
-                         :responses  {200 {:body {:resultId string?}}}
+                         :responses  {200 {:body ::service/generate-response}}
                          :summary    "Registers document plan for generation"
                          :coercion   reitit.coercion.spec/coercion
                          :middleware [muuntaja/format-request-middleware
@@ -101,7 +101,7 @@
                              :handler (swagger/create-swagger-handler)}}]
      ["/health" {:get health}]
      ["/status" {:get {:responses {200 {:body {:color string? :services coll?}}}
-                       :handler    status}}]]
+                       :handler   status}}]]
     {:data      {
                  :muuntaja   m/instance
                  :middleware [swagger/swagger-feature


Test patch:
diff --git a/api/test/api/end_to_end_test.clj b/api/test/api/end_to_end_test.clj
index 9de925f46..d78ed20a3 100644
--- a/api/test/api/end_to_end_test.clj
+++ b/api/test/api/end_to_end_test.clj
@@ -10,6 +10,12 @@
   (is (= {"test" []} (generate-text-bulk {:document-plan-name "Untitled plan"
                                           :data-rows          {"test" {}}}))))
 
+(deftest ^:integration non-async-text-generation
+  (is (= ["Building Search Applications is authored by Manu Konchady."]
+         (:variants (generate-text {:document-plan-name "Authorship"
+                                    :data-file-name     "books.csv"
+                                    :async              false})))))
+
 (deftest ^:integration text-generation-with-simple-plan
   (is (= ["Building Search Applications is authored by Manu Konchady."]
          (generate-text {:document-plan-name "Authorship"
diff --git a/api/test/api/test_utils.clj b/api/test/api/test_utils.clj
index a5a9a6a70..c45a0e7bb 100644
--- a/api/test/api/test_utils.clj
+++ b/api/test/api/test_utils.clj
@@ -69,14 +69,15 @@
           (< 10 retry-count) (throw (Exception. "Result was not ready after 10 seconds."))
           :else (do (Thread/sleep 1000) (recur (inc retry-count) (request!))))))))
 
-(defn generate-text [{:keys [document-plan-name data-file-name reader-flags]}]
-  (let [{:keys [status body]} (q "/nlg/" :post {:documentPlanId   (load-document-plan document-plan-name)
-                                                :readerFlagValues (or reader-flags {"Eng" true})
-                                                :dataId           (if (some? data-file-name)
-                                                                    (load-data-file data-file-name)
-                                                                    "")})]
+(defn generate-text [{:keys [document-plan-name data-file-name reader-flags async]}]
+  (let [{:keys [status body]} (q "/nlg/" :post (cond-> {:documentPlanId   (load-document-plan document-plan-name)
+                                                        :readerFlagValues (or reader-flags {"Eng" true})
+                                                        :dataId           (if (some? data-file-name)
+                                                                            (load-data-file data-file-name)
+                                                                            "")}
+                                                       (false? async) (assoc :async false)))]
     (when (= 200 status)
-      (get-result (:resultId body)))))
+      (if (false? async) body (get-result (:resultId body))))))
 
 (defn generate-text-bulk [{:keys [document-plan-name data-rows reader-flags]}]
   (let [{:keys [status body]} (q "/nlg/_bulk/" :post {:documentPlanId   (load-document-plan document-plan-name)