Repository: accelerated-text/accelerated-text (language: python)

Issue:
Numbers in excel files get converted to floats
![Screenshot from 2021-03-02 10-40-02](https://user-images.githubusercontent.com/20424394/109621477-aff49880-7b43-11eb-8b8a-9d150588aafe.png)
![Screenshot from 2021-03-02 10-39-57](https://user-images.githubusercontent.com/20424394/109621481-b1be5c00-7b43-11eb-9387-ddb91f162ca1.png)



Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 565a893b7..9addea72b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,26 +1,28 @@
-node_modules
-*.lock
-front-end/dist
-front-end/*.log
+**/*.env
+**/*.iml
+**/.clj-kondo/.cache/*
+**/.cpcache/
 **/.idea/*
 **/.stats/*
-api/.nrepl-port
-api/output-sam.yaml
-api/api.iml
-api/pom.xml
-api/server
-api/libsunec.so
 **/target/*
+*.gfo
+*.lock
+*.pgf
+.clj-kondo/
 .lein-failures
 .lein-repl-history
-**/*.iml
-**/*.env
-**/.cpcache/
+.lsp/
 .nrepl-port
-**/.clj-kondo/.cache/*
-*.gfo
-*.pgf
-eval/
+api/.nrepl-port
+api/api.iml
+api/libsunec.so
+api/output-sam.yaml
+api/pom.xml
+api/resources/data-files
 api/resources/dictionary
 api/resources/document-plans
-api/resources/data-files
+api/server
+eval/
+front-end/*.log
+front-end/dist
+node_modules
diff --git a/api/Makefile b/api/Makefile
index c3b0f56c2..61bc64e1d 100644
--- a/api/Makefile
+++ b/api/Makefile
@@ -1,5 +1,5 @@
 lint:
-	clojure -Sdeps '{:deps {clj-kondo {:mvn/version "RELEASE"}}}' -m clj-kondo.main --lint src test
+	clojure -Sdeps '{:deps {clj-kondo/clj-kondo {:mvn/version "RELEASE"}}}' -m clj-kondo.main --lint src test
 
 fmt-check:
 	clojure -Sdeps '{:deps {cljfmt/cljfmt {:mvn/version "0.7.0"}}}' -m cljfmt.main check
diff --git a/api/src/api/server.clj b/api/src/api/server.clj
index e6c87509f..0a36f2178 100755
--- a/api/src/api/server.clj
+++ b/api/src/api/server.clj
@@ -1,28 +1,28 @@
 (ns api.server
   (:gen-class)
-  (:require [api.config :refer [conf]]
+  (:require [acc-text.nlg.gf.service :as gf-service]
+            [api.config :refer [conf]]
+            [api.error :as errors]
             [api.graphql.core :as graphql]
             [api.nlg.service :as service]
             [api.resource :refer [response-examples]]
             [api.utils :as utils]
-            [api.error :as errors]
             [clojure.tools.logging :as log]
             [data.entities.data-files :as data-files]
-            [mount.core :refer [defstate] :as mount]
+            [mount.core :as mount :refer [defstate]]
+            [muuntaja.core :as m]
             [org.httpkit.server :as server]
-            [ring.middleware.multipart-params :as multipart-params]
+            [reitit.coercion.schema]
             [reitit.coercion.spec]
+            [reitit.dev.pretty :as pretty]
             [reitit.ring :as ring]
-            [reitit.coercion.schema]
-            [reitit.swagger :as swagger]
-            [reitit.swagger-ui :as swagger-ui]
-            [muuntaja.core :as m]
             [reitit.ring.coercion :as coercion]
-            [reitit.ring.middleware.parameters :as parameters]
             [reitit.ring.middleware.multipart :as multipart]
             [reitit.ring.middleware.muuntaja :as muuntaja]
-            [reitit.dev.pretty :as pretty]
-            [acc-text.nlg.gf.service :as gf-service]))
+            [reitit.ring.middleware.parameters :as parameters]
+            [reitit.swagger :as swagger]
+            [reitit.swagger-ui :as swagger-ui]
+            [ring.middleware.multipart-params :as multipart-params]))
 
 (def headers {"Access-Control-Allow-Origin"  "*"
               "Access-Control-Allow-Headers" "content-type, *"
diff --git a/api/src/data/entities/data_files.clj b/api/src/data/entities/data_files.clj
index ab4a56c31..5e3d5e893 100644
--- a/api/src/data/entities/data_files.clj
+++ b/api/src/data/entities/data_files.clj
@@ -1,18 +1,20 @@
 (ns data.entities.data-files
   (:require [api.config :refer [conf]]
             [clojure.data.csv :as csv]
-            [clojure.string :as str]
             [clojure.java.io :as io]
+            [clojure.string :as str]
             [clojure.tools.logging :as log]
             [data.db :as db]
-            [data.utils :as utils]
+            [data.entities.data-files.format-coercion :as format-coercion]
             [data.entities.data-files.row-selection :as row-selection]
             [data.spec.data-file :as data-file]
+            [data.utils :as utils]
             [dk.ative.docjure.spreadsheet :as excel]
             [mount.core :refer [defstate]]))
 
 (defstate data-files-db :start (db/db-access :data-files conf))
 
+
 (defn read-xlsx [content]
   (with-open [is (io/input-stream content)]
     (->> is
@@ -24,6 +26,7 @@
          (map excel/cell-seq)
          (map #(map excel/read-cell %))
          (remove #(every? nil? %))
+         (format-coercion/coerce-data-types)
          (csv/write-csv *out*)
          (with-out-str))))
 
diff --git a/api/src/data/entities/data_files/format_coercion.clj b/api/src/data/entities/data_files/format_coercion.clj
new file mode 100644
index 000000000..ac9494cdf
--- /dev/null
+++ b/api/src/data/entities/data_files/format_coercion.clj
@@ -0,0 +1,22 @@
+(ns data.entities.data-files.format-coercion
+  (:require [clojure.string :as string]))
+
+
+(defn transpose [m] (apply mapv vector m))
+
+(defn coerce-ints [coll]
+  (if (every? #(and (string/ends-with? (str %) ".0")
+                    (number? %)) coll)
+    (map int coll)
+    coll))
+
+(defn coerce-data-types
+  "When reading Excel files ints are converted to floats.
+  There might be other similar data type mismatches.
+  Detect such cases here and convert accordingly.
+  Expecting `rows` to have a header row"
+  [rows]
+  (->> rows
+       (transpose)
+       (map (fn [[head & items]] (cons head (coerce-ints items))))
+       (transpose)))


Test patch:
diff --git a/api/test/api/graphql/data_test.clj b/api/test/api/graphql/data_test.clj
index 5b0874d62..167abb4ee 100644
--- a/api/test/api/graphql/data_test.clj
+++ b/api/test/api/graphql/data_test.clj
@@ -1,8 +1,8 @@
 (ns api.graphql.data-test
-  (:require [api.test-utils :refer [q load-data-file]]
-            [api.db-fixtures :as db]
+  (:require [api.db-fixtures :as db]
+            [api.test-utils :refer [load-data-file q]]
             [clojure.string :as str]
-            [clojure.test :refer [deftest is use-fixtures testing]]
+            [clojure.test :refer [deftest is testing use-fixtures]]
             [data.entities.data-files :as data-files]
             [data.spec.data-file :as data-file]))
 
diff --git a/api/test/api/server_test.clj b/api/test/api/server_test.clj
index 9dea191b8..eba5a1f02 100644
--- a/api/test/api/server_test.clj
+++ b/api/test/api/server_test.clj
@@ -1,6 +1,6 @@
 (ns api.server-test
-  (:require [api.test-utils :refer [q]]
-            [api.db-fixtures :as fixtures]
+  (:require [api.db-fixtures :as fixtures]
+            [api.test-utils :refer [q]]
             [clojure.test :refer [deftest is testing use-fixtures]]
             [data.entities.data-files :as data-files]
             [data.entities.document-plan :as dp]))
diff --git a/api/test/data/data_files/format_coercion_test.clj b/api/test/data/data_files/format_coercion_test.clj
new file mode 100644
index 000000000..58973be30
--- /dev/null
+++ b/api/test/data/data_files/format_coercion_test.clj
@@ -0,0 +1,14 @@
+(ns data.data-files.format-coercion-test
+  (:require [clojure.java.io :as io]
+            [clojure.test :refer [deftest is]]
+            [data.entities.data-files :refer [read-xlsx]]
+            [data.entities.data-files.format-coercion :refer [coerce-ints]]))
+
+(deftest int-coercion
+  (is (= [1 2 3] (coerce-ints [1.0 2.0 3.0])))
+  (is (= [1.0 2.0 3] (coerce-ints [1.0 2.0 3])))
+  (is (= [1.0 2.0 "ab.0"] (coerce-ints [1.0 2.0 "ab.0"]))))
+
+(deftest coerce-excel-content
+  (is (= "x,y,z\na,1.0,1\nb,1.2,11\nc,1.0,111\nd,1.11,1111\n"
+         (read-xlsx (io/file "test/resources/data-files/dataformatting.xlsx")))))
diff --git a/api/test/resources/data-files/dataformatting.xlsx b/api/test/resources/data-files/dataformatting.xlsx
new file mode 100644
index 000000000..06d4b2662
Binary files /dev/null and b/api/test/resources/data-files/dataformatting.xlsx differ