Repository: chrovis/cljam
Pull request #312 (2024-06-24 08:04:43)

Implement ITF-8/LTF-8 encoder for CRAM writer
This PR introduces the implementation of the ITF-8/LTF-8 encoder as a step towards adding a CRAM writer.

Unlike the existing ITF-8 decoder, which is based on ByteBuffer, the new ITF-8 encoder is implemented using OutputStream. This makes it easier to use without needing to determine the buffer size beforehand.

While adding tests, I found that some of the existing test cases for the ITF-8 decoder used invalidly encoded ITF-8 byte sequences. This PR includes corrections for these test cases.


Patch:
diff --git a/src/cljam/io/cram/itf8.clj b/src/cljam/io/cram/itf8.clj
index cfaf45c..41a99b4 100644
--- a/src/cljam/io/cram/itf8.clj
+++ b/src/cljam/io/cram/itf8.clj
@@ -1,5 +1,6 @@
 (ns cljam.io.cram.itf8
-  (:require [cljam.io.util.byte-buffer :as bb]))
+  (:require [cljam.io.util.byte-buffer :as bb]
+            [cljam.io.util.lsb.io-stream :as lsb]))
 
 (defn decode-itf8
   "Decodes ITF-8 integer from ByteBuffer."
@@ -95,3 +96,99 @@
                   (bit-shift-left (long (bb/read-ubyte bb)) 16)
                   (bit-shift-left (long (bb/read-ubyte bb)) 8)
                   (long (bb/read-ubyte bb))))))
+
+(defn encode-itf8
+  "Encodes ITF-8 integer to OutputStream."
+  [out ^long v]
+  (cond (zero? (unsigned-bit-shift-right v 7))
+        (lsb/write-ubyte out v)
+
+        (zero? (unsigned-bit-shift-right v 14))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 8) 0x80))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 21))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 16) 0xc0))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 28))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 24) 0xe0))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        :else
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 28) 0xf0))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 20) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 12) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 4) 0xff))
+            (lsb/write-ubyte out (bit-and v 0x0f))))
+  nil)
+
+(defn encode-ltf8
+  "Encodes LTF-8 integer to OutputStream."
+  [out ^long v]
+  (cond (zero? (unsigned-bit-shift-right v 7))
+        (lsb/write-ubyte out v)
+
+        (zero? (unsigned-bit-shift-right v 14))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 8) 0x80))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 21))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 16) 0xc0))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 28))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 24) 0xe0))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 35))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 32) 0xf0))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 24) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 42))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 40) 0xf8))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 32) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 24) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 49))
+        (do (lsb/write-ubyte out (bit-or (unsigned-bit-shift-right v 48) 0xfc))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 40) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 32) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 24) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        (zero? (unsigned-bit-shift-right v 56))
+        (do (lsb/write-ubyte out 0xfe)
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 48) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 40) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 32) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 24) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff)))
+
+        :else
+        (do (lsb/write-ubyte out 0xff)
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 56) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 48) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 40) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 32) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 24) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 16) 0xff))
+            (lsb/write-ubyte out (bit-and (unsigned-bit-shift-right v 8) 0xff))
+            (lsb/write-ubyte out (bit-and v 0xff))))
+  nil)


Test patch:
diff --git a/test/cljam/io/cram/itf8_test.clj b/test/cljam/io/cram/itf8_test.clj
index 83f803a..45102b1 100644
--- a/test/cljam/io/cram/itf8_test.clj
+++ b/test/cljam/io/cram/itf8_test.clj
@@ -1,7 +1,8 @@
 (ns cljam.io.cram.itf8-test
   (:require [cljam.io.cram.itf8 :as itf8]
             [cljam.io.util.byte-buffer :as bb]
-            [clojure.test :refer [deftest are]]))
+            [clojure.test :refer [are deftest]])
+  (:import [java.io ByteArrayOutputStream]))
 
 (deftest decode-itf8-test
   (are [bs expected] (= expected
@@ -12,12 +13,12 @@
     [0x7f] 0x7f
     [0x80 0xff] 0xff
     [0xbc 0xab] 0x3cab
-    [0xc0 0x12 0x34] 0x1234
+    [0xc0 0x56 0x78] 0x5678
     [0xda 0xbc 0x23] 0x1abc23
-    [0xe0 0x12 0x34 0x56] 0x123456
+    [0xe0 0x23 0x45 0x67] 0x234567
     [0xea 0xbc 0xde 0xf0] 0xabcdef0
-    [0xfc 0xaf 0xeb 0xab 0x0e] (unchecked-int 0xcafebabe)
-    [0xff 0xff 0xff 0xff 0xff] -1))
+    [0xf1 0x23 0x45 0x67 0x08] 0x12345678
+    [0xff 0xff 0xff 0xff 0x0f] -1))
 
 (deftest decode-ltf8-test
   (are [bs expected] (= expected
@@ -28,9 +29,9 @@
     [0x7f] 0x7f
     [0x80 0xff] 0xff
     [0xbc 0xab] 0x3cab
-    [0xc0 0x12 0x34] 0x1234
+    [0xc0 0x56 0x78] 0x5678
     [0xda 0xbc 0x23] 0x1abc23
-    [0xe0 0x12 0x34 0x56] 0x123456
+    [0xe0 0x23 0x45 0x67] 0x234567
     [0xea 0xbc 0xde 0xf0] 0xabcdef0
     [0xf0 0xde 0xad 0xbe 0xef] 0xdeadbeef
     [0xf7 0xff 0xff 0xff 0xff] 0x7ffffffff
@@ -42,3 +43,43 @@
     [0xfe 0xff 0xff 0xff 0xff 0xff 0xff 0xff] 0xffffffffffffff
     [0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0] 0x123456789abcdef0
     [0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff] -1))
+
+(deftest encode-itf8-test
+  (are [v expected] (= expected
+                       (let [baos (ByteArrayOutputStream.)]
+                         (itf8/encode-itf8 baos v)
+                         (map #(bit-and % 0xff) (.toByteArray baos))))
+    0 [0x00]
+    0x7f [0x7f]
+    0xff [0x80 0xff]
+    0x3cab [0xbc 0xab]
+    0x5678 [0xc0 0x56 0x78]
+    0x1abc23 [0xda 0xbc 0x23]
+    0x234567 [0xe0 0x23 0x45 0x67]
+    0xabcdef0 [0xea 0xbc 0xde 0xf0]
+    0x12345678 [0xf1 0x23 0x45 0x67 0x08]
+    -1 [0xff 0xff 0xff 0xff 0x0f]))
+
+(deftest encode-ltf8-test
+  (are [v expected] (= expected
+                       (let [baos (ByteArrayOutputStream.)]
+                         (itf8/encode-ltf8 baos v)
+                         (map #(bit-and % 0xff) (.toByteArray baos))))
+    0 [0x00]
+    0x7f [0x7f]
+    0xff [0x80 0xff]
+    0x3cab [0xbc 0xab]
+    0x5678 [0xc0 0x56 0x78]
+    0x1abc23 [0xda 0xbc 0x23]
+    0x234567 [0xe0 0x23 0x45 0x67]
+    0xabcdef0 [0xea 0xbc 0xde 0xf0]
+    0xdeadbeef [0xf0 0xde 0xad 0xbe 0xef]
+    0x7ffffffff [0xf7 0xff 0xff 0xff 0xff]
+    0x123456789a [0xf8 0x12 0x34 0x56 0x78 0x9a]
+    0x3ffffffffff [0xfb 0xff 0xff 0xff 0xff 0xff]
+    0xbadbadbadbad [0xfc 0xba 0xdb 0xad 0xba 0xdb 0xad]
+    0x123456789abcd [0xfd 0x23 0x45 0x67 0x89 0xab 0xcd]
+    0x123456789abcde [0xfe 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde]
+    0xffffffffffffff [0xfe 0xff 0xff 0xff 0xff 0xff 0xff 0xff]
+    0x123456789abcdef0 [0xff 0x12 0x34 0x56 0x78 0x9a 0xbc 0xde 0xf0]
+    -1 [0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff]))