Repository: chrovis/cljam
Pull request #152 (2018-10-30 16:22:39)

Improve performance of TwoBitReader
#### Summary
Quick fix for #150 : Random read of 2bit is quite slower than FASTA
Now it got about **20x ~ 35x** faster than master

#### Benchmark

https://github.com/chrovis/cljam/blob/7921def33b712c099fd6cac94fe1e98ca2a94c76/bench/cljam/io/sequence_bench.clj#L7-L9

https://github.com/chrovis/cljam/blob/7921def33b712c099fd6cac94fe1e98ca2a94c76/bench/cljam/io/sequence_bench.clj#L15-L22

https://github.com/chrovis/cljam/blob/7921def33b712c099fd6cac94fe1e98ca2a94c76/bench/cljam/io/sequence_bench.clj#L28-L37

##### master
```clj
read-all-sequences-test (sequence_bench.clj:7)

  ["test-resources/fasta/medium.fa" {}]
  time: 78.420780 ms, sd: 7.932527 µs

  ["test-resources/fasta/medium.fa" {:mask? true}]
  time: 80.570329 ms, sd: 27.922591 µs

  ["test-resources/twobit/medium.2bit" {}]
  time: 121.734670 ms, sd: 1.247978 µs

  ["test-resources/twobit/medium.2bit" {:mask? true}]
  time: 125.450115 ms, sd: 19.756194 µs

read-sequence-once-test (sequence_bench.clj:15)

  ["test-resources/fasta/medium.fa" {}]
  time: 651.141680 µs, sd: 0.492259 ns

  ["test-resources/fasta/medium.fa" {:mask? true}]
  time: 688.146704 µs, sd: 1.269185 ns

  ["test-resources/twobit/medium.2bit" {}]
  time: 9.856713 ms, sd: 26.594884 ns

  ["test-resources/twobit/medium.2bit" {:mask? true}]
  time: 10.111649 ms, sd: 130.142010 ns

read-sequence-test (sequence_bench.clj:28)

  ["test-resources/fasta/medium.fa" {}]
  time: 236.486612 µs, sd: 0.017035 ns

  ["test-resources/fasta/medium.fa" {:mask? true}]
  time: 248.022497 µs, sd: 0.039226 ns

  ["test-resources/twobit/medium.2bit" {}]
  time: 9.423483 ms, sd: 11.572672 ns

  ["test-resources/twobit/medium.2bit" {:mask? true}]
  time: 9.261297 ms, sd: 28.707967 ns
```

##### PR
```clj
read-all-sequences-test (sequence_bench.clj:7)

  ["test-resources/fasta/medium.fa" {}]
  time: 79.265507 ms, sd: 1.977376 µs

  ["test-resources/fasta/medium.fa" {:mask? true}]
  time: 80.113221 ms, sd: 2.527757 µs

  ["test-resources/twobit/medium.2bit" {}]
  time: 3.511812 ms, sd: 1.397351 ns

  ["test-resources/twobit/medium.2bit" {:mask? true}]
  time: 5.465990 ms, sd: 9.133830 ns

read-sequence-once-test (sequence_bench.clj:15)

  ["test-resources/fasta/medium.fa" {}]
  time: 652.469276 µs, sd: 0.422977 ns

  ["test-resources/fasta/medium.fa" {:mask? true}]
  time: 653.855513 µs, sd: 0.440200 ns

  ["test-resources/twobit/medium.2bit" {}]
  time: 376.351904 µs, sd: 0.103506 ns

  ["test-resources/twobit/medium.2bit" {:mask? true}]
  time: 514.818163 µs, sd: 0.014057 ns

read-sequence-test (sequence_bench.clj:28)

  ["test-resources/fasta/medium.fa" {}]
  time: 240.530200 µs, sd: 0.021139 ns

  ["test-resources/fasta/medium.fa" {:mask? true}]
  time: 240.975009 µs, sd: 0.015479 ns

  ["test-resources/twobit/medium.2bit" {}]
  time: 263.236439 µs, sd: 0.010704 ns

  ["test-resources/twobit/medium.2bit" {:mask? true}]
  time: 408.126779 µs, sd: 0.025648 ns
```

#### Changes
- Add bench scripts
- Rewrite `cljam.io.twobit.reader.TwoBitReader` leveraging `java.nio.MappedByteBuffer`
- Add `read-seq-summaries` fn for quick access to `:name` and `:len` without loading indices

#### Tests

- `lein check` 🆗
- `lein test :all` 🆗
- `lein eastwood` 🆗


Patch:
diff --git a/bench/cljam/io/sequence_bench.clj b/bench/cljam/io/sequence_bench.clj
index e43975b..197e98d 100644
--- a/bench/cljam/io/sequence_bench.clj
+++ b/bench/cljam/io/sequence_bench.clj
@@ -11,3 +11,31 @@
     tcommon/medium-fa-file {:mask? true}
     tcommon/medium-twobit-file {}
     tcommon/medium-twobit-file {:mask? true}))
+
+(defbench read-sequence-once-test
+  (are [f opts]
+      (let [region {:chr "chr7",
+                    :start 10000,
+                    :end 45000}]
+        (c/quick-bench
+         (with-open [rdr (cseq/reader f)]
+           (cseq/read-sequence rdr region opts))))
+    tcommon/medium-fa-file {}
+    tcommon/medium-fa-file {:mask? true}
+    tcommon/medium-twobit-file {}
+    tcommon/medium-twobit-file {:mask? true}))
+
+(defbench read-sequence-test
+  (are [f opts]
+      (let [region {:chr "chr7",
+                    :start 10000,
+                    :end 45000}]
+        (with-open [rdr (cseq/reader f)]
+          ;; warm up
+          (cseq/read-sequence rdr region opts)
+          (c/quick-bench
+           (cseq/read-sequence rdr region opts))))
+    tcommon/medium-fa-file {}
+    tcommon/medium-fa-file {:mask? true}
+    tcommon/medium-twobit-file {}
+    tcommon/medium-twobit-file {:mask? true}))
diff --git a/src/cljam/io/fasta/core.clj b/src/cljam/io/fasta/core.clj
index f61adec..96acb3b 100644
--- a/src/cljam/io/fasta/core.clj
+++ b/src/cljam/io/fasta/core.clj
@@ -48,6 +48,14 @@
     (catch FileNotFoundException _
       (reader/load-headers (.reader rdr)))))
 
+(defn read-seq-summaries
+  "Read summaries of sequences in this FASTA file."
+  [^FASTAReader rdr]
+  (mapv
+   (fn [{:keys [name len]}]
+     {:name name, :len len})
+   (fai/get-indices @(.index-delay rdr))))
+
 (defn read-indices
   [^FASTAReader rdr]
   (fai/get-indices @(.index-delay rdr)))
@@ -95,6 +103,8 @@
     ([this region option]
      (protocols/read-sequence this region option)))
   protocols/ISequenceReader
+  (read-seq-summaries
+    [this] (read-seq-summaries this))
   (read-indices
     [this] (read-indices this))
   (read-all-sequences
diff --git a/src/cljam/io/protocols.clj b/src/cljam/io/protocols.clj
index 2b5c786..6686b05 100644
--- a/src/cljam/io/protocols.clj
+++ b/src/cljam/io/protocols.clj
@@ -57,6 +57,8 @@
     "Writes variants to thee VCF/BCF file."))
 
 (defprotocol ISequenceReader
+  (read-seq-summaries [this]
+    "Reads summaries of sequences in the file.")
   (read-indices [this]
     "Reads metadata of indexed sequences in FASTA/2BIT file.")
   (read-all-sequences [this] [this option]
diff --git a/src/cljam/io/sequence.clj b/src/cljam/io/sequence.clj
index 824cae3..1c0f7e6 100644
--- a/src/cljam/io/sequence.clj
+++ b/src/cljam/io/sequence.clj
@@ -53,9 +53,15 @@
   ([rdr] (protocols/read-all-sequences rdr))
   ([rdr option] (protocols/read-all-sequences rdr option)))
 
+(defn read-seq-summaries
+  "Returns summaries of sequences in FASTA/TwoBit file. Returns a vector of maps
+  containing `:name` and `:len`."
+  [rdr]
+  (protocols/read-seq-summaries rdr))
+
 (defn read-indices
   "Reads metadata of indexed sequences. Returns a vector of maps containing
-  :name, :len and other format-specific keys."
+  `:name`, `:len` and other format-specific keys. Forces loading all indices."
   [rdr]
   (protocols/read-indices rdr))
 
diff --git a/src/cljam/io/twobit/reader.clj b/src/cljam/io/twobit/reader.clj
index a0d2112..65764fc 100644
--- a/src/cljam/io/twobit/reader.clj
+++ b/src/cljam/io/twobit/reader.clj
@@ -1,96 +1,103 @@
 (ns cljam.io.twobit.reader
-  (:require [clojure.java.io :as cio]
-            [cljam.io.protocols :as protocols]
-            [cljam.io.util.lsb :as lsb]
+  (:require [cljam.io.protocols :as protocols]
             [cljam.util :as util])
-  (:import [java.io Closeable DataInput RandomAccessFile]
-           [java.net URL]
-           [java.nio CharBuffer]))
+  (:import [java.io Closeable]
+           [java.util TreeMap HashMap]
+           [java.nio CharBuffer ByteBuffer ByteOrder]
+           [java.nio.channels FileChannel FileChannel$MapMode]
+           [java.nio.file Paths OpenOption StandardOpenOption]))
 
-(defrecord TwoBitReader [^RandomAccessFile reader ^URL url endian file-index seq-index]
+(deftype TwoBitReader [buf url index]
   Closeable
-  (close [this]
-    (.close ^Closeable (.reader this))))
-
-(defn- read-file-header!
-  [^DataInput rdr]
-  (let [sig (.readInt rdr)
-        endian (case sig
-                 0x1A412743 :big
-                 0x4327411A :little
-                 :else nil)]
-    (when endian
-      (let [read-fn (case endian :big #(.readInt ^DataInput %) :little lsb/read-int)
-            ver (read-fn rdr)
-            nseq (read-fn rdr)
-            zero (read-fn rdr)]
-        (when (and (zero? ver) (zero? zero))
-          [endian nseq])))))
-
-(defn- read-index!
-  [^DataInput rdr endian]
-  (let [name-size (case endian
-                    :little (lsb/read-ubyte rdr)
-                    :big (.readUnsignedByte rdr))
-        name-str (case endian
-                   :little (lsb/read-string rdr name-size)
-                   :big (let [ba (byte-array name-size)]
-                          (.readFully rdr ba)
-                          (String. ba)))
-        offset (case endian
-                 :little (lsb/read-int rdr)
-                 :big (.readInt rdr))]
-    {:name name-str
-     :offset offset}))
-
-(defn- read-sequence-header!
-  [^DataInput rdr endian]
-  (let [read-fn (case endian :little lsb/read-int :big #(.readInt ^DataInput %))
-        dna-size (read-fn rdr)
-        n-block-count (read-fn rdr)
-        n-block-starts (doall (repeatedly n-block-count #(inc (read-fn rdr))))
-        n-block-sizes (doall (repeatedly n-block-count #(read-fn rdr)))
-        mask-block-count (read-fn rdr)
-        mask-block-starts (doall (repeatedly mask-block-count #(inc (read-fn rdr))))
-        mask-block-sizes (doall (repeatedly mask-block-count #(read-fn rdr)))
-        zero (read-fn rdr)]
-    {:len dna-size
-     :ambs (mapv vector n-block-starts n-block-sizes)
-     :masks (mapv vector mask-block-starts mask-block-sizes)
-     :header-offset (+ 16 (* n-block-count 4 2) (* mask-block-count 4 2))}))
-
-(def ^:const twobit-to-str
+  (close [this]))
+
+(defrecord ChromHeader [ambs masks ^long header-offset])
+
+(defrecord Chrom [name ^int len ^int offset ^int index header])
+
+(defn- ^TreeMap read-header-block! [^ByteBuffer buf]
+  (let [n-blocks (.getInt buf)
+        starts (doto (.slice buf) (.order (.order buf)))
+        _ (.position buf (+ (.position buf) (* Integer/BYTES n-blocks)))
+        m (TreeMap.)]
+    (dotimes [_ n-blocks]
+      (.put m (unchecked-inc-int (.getInt starts)) (.getInt buf)))
+    (assert (= (.size m) n-blocks))
+    m))
+
+(defn- read-sequence-header! [buf]
+  (let [amb-blocks (read-header-block! buf)
+        mask-blocks (read-header-block! buf)]
+    (ChromHeader.
+     amb-blocks
+     mask-blocks
+     (* Integer/BYTES
+        (+ 4 (* (.size amb-blocks) 2) (* (.size mask-blocks) 2))))))
+
+(defn- read-file-index! [^ByteBuffer buf ^long n-seqs]
+  (let [m (HashMap.)
+        ba (byte-array 255)]
+    (dotimes [i n-seqs]
+      (let [chr-len (Byte/toUnsignedInt (.get buf))
+            _ (.get buf ba 0 chr-len)
+            chr (String. ba 0 chr-len)
+            offset (.getInt buf)
+            _ (.mark buf)
+            _ (.position buf offset)
+            len (.getInt buf)
+            _ (.reset buf)
+            header (delay
+                    (let [buf' (.duplicate buf)]
+                      (.order buf' (.order buf))
+                      (.position buf' (+ offset Integer/BYTES))
+                      (read-sequence-header! buf')))]
+        (.put m chr (Chrom. chr len offset i header))))
+    m))
+
+(def ^:private ^"[[C" twobit-to-str
   (let [table "TCAG"]
-    (mapv
-     (fn [j] (let [i (byte (- j 128))
-                   n4 (bit-and i 2r11)
-                   n3 (bit-and (unsigned-bit-shift-right i 2) 2r11)
-                   n2 (bit-and (unsigned-bit-shift-right i 4) 2r11)
-                   n1 (bit-and (unsigned-bit-shift-right i 6) 2r11)]
-               (str (.charAt table n1) (.charAt table n2) (.charAt table n3) (.charAt table n4))))
-     (range 256))))
+    (->> 256
+         range
+         (mapv
+          (fn [j] (let [i (byte (- j 128))
+                        n4 (bit-and i 2r11)
+                        n3 (bit-and (unsigned-bit-shift-right i 2) 2r11)
+                        n2 (bit-and (unsigned-bit-shift-right i 4) 2r11)
+                        n1 (bit-and (unsigned-bit-shift-right i 6) 2r11)]
+                    (char-array [(.charAt table n1)
+                                 (.charAt table n2)
+                                 (.charAt table n3)
+                                 (.charAt table n4)]))))
+         (into-array (Class/forName "[C")))))
 
 (defn replace-ambs!
   "Replace regions of charbuffer with Ns."
-  [^CharBuffer cb ambs ^long start ^long end]
-  (doseq [[^long n-start ^long n-size] ambs]
-    (when-not (or (< end n-start) (< (+ n-start n-size -1) start))
-      (.position cb (max 0 (- n-start start)))
-      (dotimes [_ (- (min end (+ n-start n-size -1)) (max start n-start) -1)]
-        (.put cb \N)))))
+  [^CharBuffer cb ^TreeMap ambs ^long start ^long end]
+  (let [floor (or (.floorKey ambs (int start)) (int 1))]
+    (doseq [[^long n-start ^long n-size] (.subMap ambs floor (int (inc end)))]
+      (when-not (or (< end n-start) (< (+ n-start n-size -1) start))
+        (.position cb (max 0 (- n-start start)))
+        (dotimes [_ (- (min end (+ n-start n-size -1)) (max start n-start) -1)]
+          (.put cb \N))))))
 
 (defn mask!
   "Mask regions of given charbuffer."
-  [^CharBuffer cb masks ^long start ^long end]
-  (doseq [[^long m-start ^long m-size] masks]
-    (when-not (or (< end m-start) (< (+ m-start m-size -1) start))
-      (.position cb (max 0 (- m-start start)))
-      (.mark cb)
-      (let [ca (char-array (- (min end (+ m-start m-size -1)) (max start m-start) -1))]
-        (.get cb ca)
-        (.reset cb)
-        (dotimes [i (alength ca)]
-          (.put cb (char (+ (int (aget ca i)) 32)))))))) ;; lower case character
+  [^CharBuffer cb ^TreeMap masks ^long start ^long end]
+  (let [floor (or (.floorKey masks (int start)) (int 1))]
+    (doseq [[^long m-start ^long m-size] (.subMap masks floor (int (inc end)))]
+      (when-not (or (< end m-start) (< (+ m-start m-size -1) start))
+        (.position cb (max 0 (- m-start start)))
+        (.mark cb)
+        (let [ca (char-array
+                  (- (min end (+ m-start m-size -1))
+                     (max start m-start) -1))]
+          (.get cb ca)
+          (.reset cb)
+          (dotimes [i (alength ca)]
+            ;; to lower case character
+            (.put cb (unchecked-char
+                      (unchecked-add-int
+                       (unchecked-int (aget ca i)) 32)))))))))
 
 (defn ^String read-sequence
   "Reads sequence at the given region from reader.
@@ -98,28 +105,29 @@
   ([rdr region]
    (read-sequence rdr region {}))
   ([^TwoBitReader rdr {:keys [chr start end]} {:keys [mask?] :or {mask? false}}]
-   (when-let [[n {:keys [offset]}]
-              (first (filter (fn [[i {:keys [name]}]] (= name chr)) (map vector (range) (.file-index rdr))))]
-     (let [{:keys [len ambs masks header-offset]} @(nth (.seq-index rdr) n) ;; Potential seek & read.
-           start' (max 1 (or start 1))
-           end' (min len (or end len))]
+   (when-let [^Chrom c (get (.index rdr) chr)]
+     (let [start' (max 1 (or start 1))
+           end' (min (.len c) (or end (.len c)))]
        (when (<= start' end')
-         (let [start-offset (quot (dec start') 4)
+         ;; Potential seek & read.
+         (let [^ChromHeader h @(.header c)
+               start-offset (quot (dec start') 4)
                end-offset (quot (dec end') 4)
-               ba (byte-array (- end-offset start-offset -1))
-               cb (CharBuffer/allocate (inc (- end' start')))]
-           (.seek ^RandomAccessFile (.reader rdr) (+ offset header-offset start-offset))
-           (.readFully ^RandomAccessFile (.reader rdr) ba)
-           (dotimes [out-pos (inc (- end' start'))]
-             (let [ref-pos (+ out-pos start')
-                   ba-pos (- (quot (dec ref-pos) 4) start-offset)
-                   bit-pos (mod (dec ref-pos) 4)]
-               (when (<= 1 ref-pos len)
-                 (.put cb (.charAt ^String (twobit-to-str (+ (aget ba ba-pos) 128)) bit-pos)))))
-           (replace-ambs! cb ambs start' end')
-           (when mask? (mask! cb masks start' end'))
-           (.rewind cb)
-           (.toString cb)))))))
+               buf ^ByteBuffer (.buf rdr)
+               cb (CharBuffer/allocate (* 4 (inc (- end-offset start-offset))))]
+           (.position buf (+ (.offset c) (.header-offset h) start-offset))
+           (while (.hasRemaining cb)
+             (->> (unchecked-add-int 128 (.get buf))
+                  ^chars (aget twobit-to-str)
+                  (.put cb)))
+           (let [cb' (-> cb
+                         ^CharBuffer (.position (mod (dec start') 4))
+                         ^CharBuffer .slice
+                         (.limit (int (inc (- end' start')))))]
+             (replace-ambs! cb' (.ambs h) start' end')
+             (when mask? (mask! cb' (.masks h) start' end'))
+             (.rewind cb')
+             (.toString cb'))))))))
 
 (defn- read-all-sequences*
   [rdr chrs option]
@@ -135,15 +143,26 @@
   ([rdr]
    (read-all-sequences rdr {}))
   ([^TwoBitReader rdr option]
-   (read-all-sequences* rdr (.file-index rdr) option)))
+   (read-all-sequences* rdr (sort-by :index (vals (.index rdr))) option)))
+
+(defn read-seq-summaries
+  "Reads summaries of sequences in this 2bit file."
+  [^TwoBitReader rdr]
+  (mapv
+   (fn [^Chrom c]
+     {:name (.name c), :len (.len c)})
+   (sort-by :index (vals (.index rdr)))))
 
 (defn read-indices
-  "Reads metadata of indexed sequences."
+  "Reads metadata of indexed sequences. Forces loading all indices."
   [^TwoBitReader rdr]
   (mapv
-   (fn [fi si] (merge fi @si))
-   (.file-index rdr)
-   (.seq-index rdr)))
+   (fn [{:keys [name len offset header]}]
+     (let [{:keys [ambs header-offset masks]} @header]
+       {:name name, :len len, :offset offset,
+        :ambs (into {} ambs), :masks (into {} masks)
+        :header-offset header-offset}))
+   (sort-by :index (vals (.index rdr)))))
 
 (extend-type TwoBitReader
   protocols/IReader
@@ -153,6 +172,8 @@
     ([this option] (protocols/read-all-sequences this option)))
   (indexed? [_] true)
   protocols/ISequenceReader
+  (read-seq-summaries
+    [this] (read-seq-summaries this))
   (read-indices
     [this] (read-indices this))
   (read-all-sequences
@@ -172,24 +193,31 @@
      (read-sequence this region option))))
 
 (defn ^TwoBitReader reader
-  "Returns .2bit file reader of f."
-  [^String f]
-  (let [abs-f (.getAbsolutePath (cio/file f))
-        rdr (RandomAccessFile. abs-f "r")
-        [endian nseq] (read-file-header! rdr)]
-    (when (and endian nseq)
-      (let [indices (vec (repeatedly nseq #(read-index! rdr endian)))
-            seq-indices (mapv (fn [{:keys [offset] :as m}]
-                                (delay
-                                 (with-open [raf (RandomAccessFile. abs-f "r")]
-                                   (.seek raf offset)
-                                   (read-sequence-header! raf endian))))
-                              indices)]
-        (TwoBitReader. rdr (util/as-url abs-f) endian indices seq-indices)))))
+  [f]
+  (let [url (util/as-url f)]
+    (with-open [ch (-> url
+                       .toURI
+                       Paths/get
+                       (FileChannel/open
+                        (into-array OpenOption [StandardOpenOption/READ])))]
+      (let [buf (.map ch FileChannel$MapMode/READ_ONLY 0 (.size ch))
+            _ (.order buf (case (.getInt buf)
+                            0x1A412743 ByteOrder/BIG_ENDIAN
+                            0x4327411A ByteOrder/LITTLE_ENDIAN))
+            version (.getInt buf)
+            n-seqs (.getInt buf)
+            zero (.getInt buf)]
+        (when-not (zero? version)
+          (throw (ex-info "Version number must be zero."
+                          {:input f, :url url, :version version})))
+        (when-not (zero? zero)
+          (throw (ex-info "sequenceCount must be followed by zero."
+                          {:input f, :url url, :zero zero})))
+        (TwoBitReader. buf url (read-file-index! buf n-seqs))))))
 
 (defn ^TwoBitReader clone-reader
   "Clones .2bit reader sharing persistent objects."
   [^TwoBitReader rdr]
-  (let [url (.url rdr)
-        raf (RandomAccessFile. (cio/as-file url) "r")]
-    (TwoBitReader. raf url (.endian rdr) (.file-index rdr) (.seq-index rdr))))
+  (let [buf (doto (.duplicate ^ByteBuffer (.buf rdr))
+              (.order (.order ^ByteBuffer (.buf rdr))))]
+    (TwoBitReader. buf (.url rdr) (.index rdr))))


Test patch:
diff --git a/test/cljam/io/sequence_test.clj b/test/cljam/io/sequence_test.clj
index dd668d0..a6ffb3d 100644
--- a/test/cljam/io/sequence_test.clj
+++ b/test/cljam/io/sequence_test.clj
@@ -28,6 +28,36 @@
       test-fai-file
       (Object.))))
 
+(deftest multithread-reader-test
+  (with-open [f (cseq/reader medium-fa-file)
+              t (cseq/reader medium-twobit-file)]
+    (let [xs (cseq/read-seq-summaries f)]
+      (is (->> (repeatedly
+                #(let [{:keys [name len]} (rand-nth xs)
+                       [s e] (sort [(inc (rand-int len))
+                                    (inc (rand-int len))])]
+                   {:chr name :start s :end e}))
+               (take 100)
+               (pmap
+                (fn [region]
+                  (with-open [fc (cseq/reader f)
+                              tc (cseq/reader t)]
+                    (= (cseq/read-sequence fc region)
+                       (cseq/read-sequence tc region)))))
+               (every? true?))))))
+
+(deftest read-seq-summaries-test
+  (testing "fasta"
+    (with-open [rdr (cseq/reader test-fa-file)]
+      (is (= (cseq/read-seq-summaries rdr)
+             [{:name "ref", :len 45}
+              {:name "ref2", :len 40}]))))
+  (testing "twobit"
+    (with-open [rdr (cseq/reader test-twobit-file)]
+      (is (= (cseq/read-seq-summaries rdr)
+             [{:name "ref", :len 45}
+              {:name "ref2", :len 40}])))))
+
 (deftest read-indices-test
   (testing "fasta"
     (with-open [rdr (cseq/reader test-fa-file)]
@@ -37,8 +67,8 @@
   (testing "twobit"
     (with-open [rdr (cseq/reader test-twobit-file)]
       (is (= (cseq/read-indices rdr)
-             [{:name "ref", :len 45, :offset 33, :ambs [], :masks [], :header-offset 16}
-              {:name "ref2", :len 40, :offset 61, :ambs [], :masks [[1 40]], :header-offset 24}])))))
+             [{:name "ref", :len 45, :offset 33, :ambs {}, :masks {}, :header-offset 16}
+              {:name "ref2", :len 40, :offset 61, :ambs {}, :masks {1 40}, :header-offset 24}])))))
 
 (deftest read-sequence-test
   (with-open [fa-rdr (cseq/reader test-fa-file)