Repository: accelerated-text/accelerated-text (language: python)

Issue:
Prevent duplicate file creation
When files are put in data-files folder and datomic is used as storage, after every restart, duplicate files get created.


Discussion / hints:

Fix patch:
diff --git a/api/resources/datomic-schema/2019-10-28-initial-schema.edn b/api/resources/datomic-schema/2019-10-28-initial-schema.edn
index 3c35c6efc..315f5c6f6 100644
--- a/api/resources/datomic-schema/2019-10-28-initial-schema.edn
+++ b/api/resources/datomic-schema/2019-10-28-initial-schema.edn
@@ -1,18 +1,21 @@
 {:acc-text/data-file-schema
- {:txes [[{:db/ident       :data-file/id
+ {:txes [[{:db/ident       :data.spec.data-file/id
+           :db/unique      :db.unique/identity
            :db/valueType   :db.type/string
            :db/cardinality :db.cardinality/one
-           :db/doc         "Public Id of the Data File"}
-
-          {:db/ident       :data-file/filename
+           :db/doc         "Public id of the data file"}
+          {:db/ident       :data.spec.data-file/name
            :db/valueType   :db.type/string
            :db/cardinality :db.cardinality/one
-           :db/doc         "File name of the Data File"}
-
-          {:db/ident       :data-file/content
+           :db/doc         "Name of the data file"}
+          {:db/ident       :data.spec.data-file/timestamp
+           :db/valueType   :db.type/long
+           :db/cardinality :db.cardinality/one
+           :db/doc         "Timestamp when file was registered"}
+          {:db/ident       :data.spec.data-file/content
            :db/valueType   :db.type/string
            :db/cardinality :db.cardinality/one
-           :db/doc         "Content of the Data File"}]
+           :db/doc         "Content of the data file"}]
 
          [{:db/ident       :data.spec.reader-model/code
            :db/unique      :db.unique/identity
diff --git a/api/src/data/datomic/entities/data_files.clj b/api/src/data/datomic/entities/data_files.clj
index e259b6346..f01cd3b65 100644
--- a/api/src/data/datomic/entities/data_files.clj
+++ b/api/src/data/datomic/entities/data_files.clj
@@ -1,28 +1,24 @@
 (ns data.datomic.entities.data-files
-  (:require [datomic.api :as d]))
+  (:require [data.spec.data-file :as data-file]
+            [datomic.api :as d]))
+
+(def pattern [::data-file/id
+              ::data-file/name
+              ::data-file/timestamp
+              ::data-file/content])
 
 (defn transact-item [conn key data-item]
-  @(d/transact conn [{:data-file/id       key
-                      :data-file/filename (:filename data-item)
-                      :data-file/content  (:content data-item)}]))
+  @(d/transact conn [(assoc data-item ::data-file/id key)]))
 
 (defn pull-entity [conn key]
-  (let [data-file (ffirst (d/q '[:find (pull ?e [*])
-                                 :in $ ?key
-                                 :where
-                                 [?e :data-file/id ?key]]
-                               (d/db conn)
-                               key))]
-    (when data-file
-      {:id       (:data-file/id data-file)
-       :filename (:data-file/filename data-file)
-       :content  (:data-file/content data-file)})))
+  (d/pull (d/db conn) pattern [::data-file/id key]))
 
 (defn pull-n [conn limit]
-  (let [resp (map first (d/q '[:find (pull ?e [*])
-                               :where [?e :data-file/id]]
-                             (d/db conn)))]
+  (map first (take limit (d/q '[:find (pull ?e pattern)
+                                :in $ pattern
+                                :where
+                                [?e ::data-file/id]]
+                              (d/db conn) pattern))))
 
-    (map (fn [df] {:id       (:data-file/id df)
-                   :filename (:data-file/filename df)
-                   :content  (:data-file/content df)}) (take limit resp))))
+(defn delete [conn key]
+  @(d/transact conn [[:db.fn/retractEntity [::data-file/id key]]]))
diff --git a/api/src/data/datomic/impl.clj b/api/src/data/datomic/impl.clj
index 6855a1d34..c3479afd1 100644
--- a/api/src/data/datomic/impl.clj
+++ b/api/src/data/datomic/impl.clj
@@ -44,6 +44,8 @@
   (data-files/pull-entity conn key))
 (defmethod pull-n :data-files [_ limit]
   (data-files/pull-n conn limit))
+(defmethod delete :data-files [_ key]
+  (data-files/delete conn key))
 
 (defmethod transact-item :dictionary [_ key data-item]
   (dictionary/transact-item conn key data-item))
diff --git a/api/src/data/entities/data_files.clj b/api/src/data/entities/data_files.clj
index f2c1f3eb0..7d49e4d60 100644
--- a/api/src/data/entities/data_files.clj
+++ b/api/src/data/entities/data_files.clj
@@ -6,64 +6,67 @@
             [clojure.tools.logging :as log]
             [data.db :as db]
             [data.utils :as utils]
-            [mount.core :refer [defstate]]
-            [dk.ative.docjure.spreadsheet :as excel]))
+            [data.spec.data-file :as data-file]
+            [dk.ative.docjure.spreadsheet :as excel]
+            [mount.core :refer [defstate]]))
 
 (defstate data-files-db :start (db/db-access :data-files conf))
 
-(defn binary->csv [{content :content :as data-file}]
-  (assoc data-file :content (cond-> content (not (string? content)) (slurp))))
+(defn read-xlsx [content]
+  (with-open [is (io/input-stream content)]
+    (->> is
+         (excel/load-workbook-from-stream)
+         (excel/sheet-seq)
+         (first)
+         (excel/row-seq)
+         (remove nil?)
+         (map excel/cell-seq)
+         (map #(map excel/read-cell %))
+         (remove #(every? nil? %))
+         (csv/write-csv *out*)
+         (with-out-str))))
 
-(defn xlsx->csv [data-file]
-  (let [content (:content data-file)]
-    (assoc data-file :content (->> (io/input-stream content)
-                                   (excel/load-workbook-from-stream)
-                                   (excel/sheet-seq)
-                                   (first)
-                                   (excel/row-seq)
-                                   (remove nil?)
-                                   (map excel/cell-seq)
-                                   (map #(map excel/read-cell %))
-                                   (remove #(every? nil? %))
-                                   (csv/write-csv *out*)
-                                   (with-out-str)))))
+(defn convert-file [{:keys [filename content]}]
+  (cond
+    (instance? String content) content
+    (str/ends-with? filename ".xlsx") (read-xlsx content)
+    :else (slurp content)))
 
-(defn convert-file [data-file]
-  (let [file-name (:filename data-file)
-        format    (cond
-                   (str/ends-with? file-name ".csv") :csv
-                   (str/ends-with? file-name ".xlsx") :xlsx
-                   :else                          :unknown)]
-    (case format
-          :csv     (binary->csv data-file)
-          :xlsx    (xlsx->csv data-file)
-          :unknown data-file)))
+(defn read-data-file [key]
+  (log/infof "Searching for data file: `%s`" key)
+  (db/read! data-files-db key))
+
+(defn delete-data-file! [key]
+  (log/infof "Deleting data file: `%s`" key)
+  (db/delete! data-files-db key))
 
 (defn store!
   "Expected keys are :filename and :content everything else is optional"
-  [data-file]
-  (let [id (utils/gen-uuid)]
-    (log/infof "Storing `%s` with id: `%s`" (:filename data-file) id)
-    (db/write! data-files-db id (convert-file data-file))
-    id))
-
-(defn read-data-file [key]
-  (db/read! data-files-db key))
+  [{filename :filename :as data-file}]
+  (when (some? (read-data-file filename))
+    (delete-data-file! filename))
+  (log/infof "Storing data file: `%s`" filename)
+  (db/write! data-files-db filename
+             #::data-file{:name      filename
+                          :timestamp (utils/ts-now)
+                          :content   (convert-file data-file)})
+  filename)
 
 (defn parse-data
-  ([data] (parse-data data 0 Integer/MAX_VALUE))
-  ([data offset limit]
-   (when (some? data)
-     (let [[header & rows] (->> (get data :content) (csv/read-csv) (map #(map str/trim %)))]
-       {:filename (get data :filename)
-        :header   (vec header)
-        :rows     (take limit (drop offset rows))
-        :offset   offset
-        :limit    limit
-        :total    (count rows)}))))
+  ([data-file]
+   (parse-data data-file 0 Integer/MAX_VALUE))
+  ([{::data-file/keys [id name content]} offset limit]
+   (let [[header & rows] (map #(map str/trim %) (cond-> content (some? content) (csv/read-csv)))]
+     {:id       id
+      :filename name
+      :header   (vec header)
+      :rows     (take limit (drop offset rows))
+      :offset   offset
+      :limit    limit
+      :total    (count rows)})))
 
-(defn fetch [id offset limit]
-  (when-let [{:keys [filename header rows total]} (some-> id (read-data-file) (parse-data offset limit))]
+(defn read-content [data offset limit]
+  (let [{:keys [id filename header rows total]} (parse-data data offset limit)]
     {:id           id
      :fileName     filename
      :fieldNames   header
@@ -82,19 +85,27 @@
      :recordLimit  limit
      :recordCount  total}))
 
-(defn listing [offset limit recordOffset recordLimit]
-  (let [data-files (db/list! data-files-db Integer/MAX_VALUE)]
-    {:dataFiles  (map (fn [{id :id}]
-                        (fetch id recordOffset recordLimit))
-                      (->> data-files (drop offset) (take limit)))
-     :offset     offset
-     :limit      limit
-     :totalCount (count data-files)}))
+(defn fetch [id offset limit]
+  (some-> id
+          (read-data-file)
+          (read-content offset limit)))
+
+(defn listing
+  ([] (listing 0 Integer/MAX_VALUE 0 Integer/MAX_VALUE))
+  ([offset limit recordOffset recordLimit]
+   (let [data-files (db/list! data-files-db Integer/MAX_VALUE)]
+     {:dataFiles  (->> data-files
+                       (drop offset)
+                       (take limit)
+                       (map #(read-content % recordOffset recordLimit)))
+      :offset     offset
+      :limit      limit
+      :totalCount (count data-files)})))
 
 (defn data-file-path []
   (or (System/getenv "DATA_FILES") "resources/data-files"))
 
 (defn initialize []
-  (doseq [f (utils/list-files (data-file-path))]
+  (doseq [f (utils/list-files-in-dir (data-file-path))]
     (store! {:filename (.getName f)
              :content  f})))
diff --git a/api/src/data/spec/data_file.clj b/api/src/data/spec/data_file.clj
new file mode 100644
index 000000000..6b54f244c
--- /dev/null
+++ b/api/src/data/spec/data_file.clj
@@ -0,0 +1,10 @@
+(ns data.spec.data-file
+  (:require [clojure.spec.alpha :as s]))
+
+(s/def ::id string?)
+
+(s/def ::name string?)
+
+(s/def ::timestamp inst?)
+
+(s/def ::content any?)
diff --git a/api/src/data/utils.clj b/api/src/data/utils.clj
index 7a86629cc..a4e3c7deb 100644
--- a/api/src/data/utils.clj
+++ b/api/src/data/utils.clj
@@ -3,7 +3,8 @@
             [clojure.edn :as edn]
             [clojure.java.io :as io]
             [jsonista.core :as json])
-  (:import (java.io File PushbackReader)
+  (:import (clojure.lang PersistentHashSet)
+           (java.io File PushbackReader)
            (java.util UUID)
            (java.time Instant)))
 
@@ -49,11 +50,27 @@
     (cond-> filename
             (not= index -1) (subs 0 index))))
 
-(defn list-files [path]
-  (some->> path
-           (io/file)
-           (file-seq)
-           (filter #(.isFile ^File %))))
+(defn file-with-ext
+  [^File f ^PersistentHashSet ext]
+  (and (some? f) (.isFile f)
+       (or (empty? ext)
+           (contains? ext (get-ext f)))))
+
+(defn list-files
+  ([path] (list-files path #{}))
+  ([path ^PersistentHashSet ext]
+   (some->> path
+            (io/file)
+            (file-seq)
+            (filter #(file-with-ext % ext)))))
+
+(defn list-files-in-dir
+  ([path] (list-files-in-dir path #{}))
+  ([path ^PersistentHashSet ext]
+   (some->> path
+            (io/file)
+            (.listFiles)
+            (filter #(file-with-ext % ext)))))
 
 (defn list-directories [path]
   (some->> path


Test patch:
diff --git a/api/docker-compose.test.yml b/api/docker-compose.test.yml
index 00198cbaf..1e96d9999 100644
--- a/api/docker-compose.test.yml
+++ b/api/docker-compose.test.yml
@@ -16,6 +16,8 @@ services:
       DOCUMENT_PLANS: "/opt/document-plans"
     command: ["clojure", "-A:dev:test", "-i", "integration"]
     volumes:
+      - ./resources/schema.graphql:/opt/schema.graphql
+      - ./resources/datomic-schema:/opt/datomic-schema
       - ./test/resources/data-files:/opt/data-files
       - ./test/resources/dictionary:/opt/dictionary
       - ./test/resources/document-plans:/opt/document-plans
diff --git a/api/test/api/datomic_fixtures.clj b/api/test/api/datomic_fixtures.clj
index 1a2124c95..1131fef16 100644
--- a/api/test/api/datomic_fixtures.clj
+++ b/api/test/api/datomic_fixtures.clj
@@ -27,7 +27,7 @@
           {#'api.config/conf
            {:start (fn []
                      {:db-implementation :datomic
-                      :enabled-languages #{"Eng"}})}
+                      :enabled-languages #{"Eng" "Ger"}})}
            #'data.entities.reader-model/language-conf
            {:start (fn []
                      (mapv data.entities.reader-model/update!
diff --git a/api/test/api/graphql/data_test.clj b/api/test/api/graphql/data_test.clj
index c0bf79c69..5b0874d62 100644
--- a/api/test/api/graphql/data_test.clj
+++ b/api/test/api/graphql/data_test.clj
@@ -3,7 +3,8 @@
             [api.db-fixtures :as db]
             [clojure.string :as str]
             [clojure.test :refer [deftest is use-fixtures testing]]
-            [data.entities.data-files :as data-files]))
+            [data.entities.data-files :as data-files]
+            [data.spec.data-file :as data-file]))
 
 (use-fixtures :each db/clean-db)
 
@@ -49,5 +50,5 @@
   (testing "Read books.csv headers"
     (let [data-file-id (load-data-file "books.csv")
           result (data-files/read-data-file data-file-id)
-          headers (-> result (get :content) (str/split-lines) (first) (str/split #",") (set))]
+          headers (-> result (get ::data-file/content) (str/split-lines) (first) (str/split #",") (set))]
       (is (= #{"authors" "categories" "pageCount" "publishedDate" "publisher" "subtitle" "title"} headers)))))