Repository: gleam-lang/gleam (Rust)

Issue:
Allow endianness specifiers for pattern matching on codepoints
I was decoding some UTF-16 data in Gleam, but it was little-endian encoded, which means I could not use the `utf16_codepoint` segment for bit array pattern matching, because it doesn't support the `little` option.
This should be straightforward to implement as Erlang supports this out of the box, and utf codepoint matching is not yet implemented on JavaScript.


Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 190e2986f25..7bcf00a96d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -196,6 +196,10 @@
 
   ([Louis Pilfold](https://github.com/lpil))
 
+- The compiler now allows using bit array options to specify endianness when
+  construction or pattern matching on bit arrays.
+  ([Surya Rose](https://github.com/GearsDatapacks))
+
 ### Build tool
 
 - `gleam update`, `gleam deps update`, and `gleam deps download` will now print
diff --git a/compiler-core/src/bit_array.rs b/compiler-core/src/bit_array.rs
index be22acaaa2d..027a52539a9 100644
--- a/compiler-core/src/bit_array.rs
+++ b/compiler-core/src/bit_array.rs
@@ -219,10 +219,20 @@ where
         }
     }
 
-    // Endianness is only valid for int, utf16, utf32 and float
+    // Endianness is only valid for int, utf16, utf16_codepoint, utf32,
+    // utf32_codepoint and float
     match categories {
         SegmentOptionCategories {
-            type_: None | Some(Int { .. } | Utf16 { .. } | Utf32 { .. } | Float { .. }),
+            type_:
+                None
+                | Some(
+                    Int { .. }
+                    | Utf16 { .. }
+                    | Utf32 { .. }
+                    | Utf16Codepoint { .. }
+                    | Utf32Codepoint { .. }
+                    | Float { .. },
+                ),
             ..
         } => {}
 


Test patch:
diff --git a/compiler-core/src/erlang/tests/bit_arrays.rs b/compiler-core/src/erlang/tests/bit_arrays.rs
index e06a8157a71..664053e3cc9 100644
--- a/compiler-core/src/erlang/tests/bit_arrays.rs
+++ b/compiler-core/src/erlang/tests/bit_arrays.rs
@@ -209,3 +209,49 @@ fn identity(x) {
 "
     );
 }
+
+#[test]
+fn utf16_codepoint_little_endian() {
+    assert_erl!(
+        "
+pub fn go(codepoint) {
+  <<codepoint:utf16_codepoint-little>>
+}
+"
+    );
+}
+
+#[test]
+fn utf32_codepoint_little_endian() {
+    assert_erl!(
+        "
+pub fn go(codepoint) {
+  <<codepoint:utf32_codepoint-little>>
+}
+"
+    );
+}
+
+#[test]
+fn pattern_match_utf16_codepoint_little_endian() {
+    assert_erl!(
+        "
+pub fn go(x) {
+  let assert <<codepoint:utf16_codepoint-little>> = x
+  codepoint
+}
+"
+    );
+}
+
+#[test]
+fn pattern_match_utf32_codepoint_little_endian() {
+    assert_erl!(
+        "
+pub fn go(x) {
+  let assert <<codepoint:utf32_codepoint-little>> = x
+  codepoint
+}
+"
+    );
+}
diff --git a/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf16_codepoint_little_endian.snap b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf16_codepoint_little_endian.snap
new file mode 100644
index 00000000000..60980075d0e
--- /dev/null
+++ b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf16_codepoint_little_endian.snap
@@ -0,0 +1,37 @@
+---
+source: compiler-core/src/erlang/tests/bit_arrays.rs
+expression: "\npub fn go(x) {\n  let assert <<codepoint:utf16_codepoint-little>> = x\n  codepoint\n}\n"
+---
+----- SOURCE CODE
+
+pub fn go(x) {
+  let assert <<codepoint:utf16_codepoint-little>> = x
+  codepoint
+}
+
+
+----- COMPILED ERLANG
+-module(my@mod).
+-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
+-define(FILEPATH, "project/test/my/mod.gleam").
+-export([go/1]).
+
+-file("project/test/my/mod.gleam", 2).
+-spec go(bitstring()) -> integer().
+go(X) ->
+    Codepoint@1 = case X of
+        <<Codepoint/utf16-little>> -> Codepoint;
+        _assert_fail ->
+            erlang:error(#{gleam_error => let_assert,
+                        message => <<"Pattern match failed, no pattern matched the value."/utf8>>,
+                        file => <<?FILEPATH/utf8>>,
+                        module => <<"my/mod"/utf8>>,
+                        function => <<"go"/utf8>>,
+                        line => 3,
+                        value => _assert_fail,
+                        start => 18,
+                        'end' => 69,
+                        pattern_start => 29,
+                        pattern_end => 65})
+    end,
+    Codepoint@1.
diff --git a/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf32_codepoint_little_endian.snap b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf32_codepoint_little_endian.snap
new file mode 100644
index 00000000000..ae1d0df4bb6
--- /dev/null
+++ b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf32_codepoint_little_endian.snap
@@ -0,0 +1,37 @@
+---
+source: compiler-core/src/erlang/tests/bit_arrays.rs
+expression: "\npub fn go(x) {\n  let assert <<codepoint:utf32_codepoint-little>> = x\n  codepoint\n}\n"
+---
+----- SOURCE CODE
+
+pub fn go(x) {
+  let assert <<codepoint:utf32_codepoint-little>> = x
+  codepoint
+}
+
+
+----- COMPILED ERLANG
+-module(my@mod).
+-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
+-define(FILEPATH, "project/test/my/mod.gleam").
+-export([go/1]).
+
+-file("project/test/my/mod.gleam", 2).
+-spec go(bitstring()) -> integer().
+go(X) ->
+    Codepoint@1 = case X of
+        <<Codepoint/utf32-little>> -> Codepoint;
+        _assert_fail ->
+            erlang:error(#{gleam_error => let_assert,
+                        message => <<"Pattern match failed, no pattern matched the value."/utf8>>,
+                        file => <<?FILEPATH/utf8>>,
+                        module => <<"my/mod"/utf8>>,
+                        function => <<"go"/utf8>>,
+                        line => 3,
+                        value => _assert_fail,
+                        start => 18,
+                        'end' => 69,
+                        pattern_start => 29,
+                        pattern_end => 65})
+    end,
+    Codepoint@1.
diff --git a/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf16_codepoint_little_endian.snap b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf16_codepoint_little_endian.snap
new file mode 100644
index 00000000000..0ba150a91e6
--- /dev/null
+++ b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf16_codepoint_little_endian.snap
@@ -0,0 +1,21 @@
+---
+source: compiler-core/src/erlang/tests/bit_arrays.rs
+expression: "\npub fn go(codepoint) {\n  <<codepoint:utf16_codepoint-little>>\n}\n"
+---
+----- SOURCE CODE
+
+pub fn go(codepoint) {
+  <<codepoint:utf16_codepoint-little>>
+}
+
+
+----- COMPILED ERLANG
+-module(my@mod).
+-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
+-define(FILEPATH, "project/test/my/mod.gleam").
+-export([go/1]).
+
+-file("project/test/my/mod.gleam", 2).
+-spec go(integer()) -> bitstring().
+go(Codepoint) ->
+    <<Codepoint/utf16-little>>.
diff --git a/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf32_codepoint_little_endian.snap b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf32_codepoint_little_endian.snap
new file mode 100644
index 00000000000..72c4ebdc10e
--- /dev/null
+++ b/compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf32_codepoint_little_endian.snap
@@ -0,0 +1,21 @@
+---
+source: compiler-core/src/erlang/tests/bit_arrays.rs
+expression: "\npub fn go(codepoint) {\n  <<codepoint:utf32_codepoint-little>>\n}\n"
+---
+----- SOURCE CODE
+
+pub fn go(codepoint) {
+  <<codepoint:utf32_codepoint-little>>
+}
+
+
+----- COMPILED ERLANG
+-module(my@mod).
+-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
+-define(FILEPATH, "project/test/my/mod.gleam").
+-export([go/1]).
+
+-file("project/test/my/mod.gleam", 2).
+-spec go(integer()) -> bitstring().
+go(Codepoint) ->
+    <<Codepoint/utf32-little>>.
diff --git a/compiler-core/src/javascript/tests/bit_arrays.rs b/compiler-core/src/javascript/tests/bit_arrays.rs
index f34d2770a2a..35296eaf91f 100644
--- a/compiler-core/src/javascript/tests/bit_arrays.rs
+++ b/compiler-core/src/javascript/tests/bit_arrays.rs
@@ -2539,3 +2539,25 @@ pub fn main() {
 "
     );
 }
+
+#[test]
+fn utf16_codepoint_little_endian() {
+    assert_js!(
+        "
+pub fn go(codepoint) {
+  <<codepoint:utf16_codepoint-little>>
+}
+"
+    );
+}
+
+#[test]
+fn utf32_codepoint_little_endian() {
+    assert_js!(
+        "
+pub fn go(codepoint) {
+  <<codepoint:utf32_codepoint-little>>
+}
+"
+    );
+}
diff --git a/compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf16_codepoint_little_endian.snap b/compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf16_codepoint_little_endian.snap
new file mode 100644
index 00000000000..5f6bf5833e4
--- /dev/null
+++ b/compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf16_codepoint_little_endian.snap
@@ -0,0 +1,17 @@
+---
+source: compiler-core/src/javascript/tests/bit_arrays.rs
+expression: "\npub fn go(codepoint) {\n  <<codepoint:utf16_codepoint-little>>\n}\n"
+---
+----- SOURCE CODE
+
+pub fn go(codepoint) {
+  <<codepoint:utf16_codepoint-little>>
+}
+
+
+----- COMPILED JAVASCRIPT
+import { toBitArray, codepointToUtf16 } from "../gleam.mjs";
+
+export function go(codepoint) {
+  return toBitArray([codepointToUtf16(codepoint, false)]);
+}
diff --git a/compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf32_codepoint_little_endian.snap b/compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf32_codepoint_little_endian.snap
new file mode 100644
index 00000000000..07cf499c13a
--- /dev/null
+++ b/compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf32_codepoint_little_endian.snap
@@ -0,0 +1,17 @@
+---
+source: compiler-core/src/javascript/tests/bit_arrays.rs
+expression: "\npub fn go(codepoint) {\n  <<codepoint:utf32_codepoint-little>>\n}\n"
+---
+----- SOURCE CODE
+
+pub fn go(codepoint) {
+  <<codepoint:utf32_codepoint-little>>
+}
+
+
+----- COMPILED JAVASCRIPT
+import { toBitArray, codepointToUtf32 } from "../gleam.mjs";
+
+export function go(codepoint) {
+  return toBitArray([codepointToUtf32(codepoint, false)]);
+}
diff --git a/test/language/test/language_test.gleam b/test/language/test/language_test.gleam
index a6c6c016f61..fe3e8495339 100644
--- a/test/language/test/language_test.gleam
+++ b/test/language/test/language_test.gleam
@@ -1094,6 +1094,20 @@ fn bit_array_target_tests() -> List(Test) {
           h
         })
       }),
+    "pattern-match UTF-16 codepoint little-endian"
+      |> example(fn() {
+        assert_equal(ffi.utf_codepoint(127_757), {
+          let assert <<codepoint:utf16_codepoint-little>> = <<"🌍":utf16-little>>
+          codepoint
+        })
+      }),
+    "pattern-match UTF-32 codepoint little-endian"
+      |> example(fn() {
+        assert_equal(ffi.utf_codepoint(127_757), {
+          let assert <<codepoint:utf32_codepoint-little>> = <<"🌍":utf32-little>>
+          codepoint
+        })
+      }),
   ]
 }
 
@@ -1707,6 +1721,14 @@ fn non_utf8_string_bit_array_tests() -> List(Test) {
           <<codepoint:utf16_codepoint>>
         })
       }),
+    "UTF-16 codepoint little-endian"
+      |> example(fn() {
+        assert_equal(<<60, 216, 13, 223>>, {
+          // 🌍
+          let codepoint = ffi.utf_codepoint(127_757)
+          <<codepoint:utf16_codepoint-little>>
+        })
+      }),
     "UTF-32 codepoint"
       |> example(fn() {
         assert_equal(<<0, 1, 243, 13>>, {
@@ -1715,6 +1737,14 @@ fn non_utf8_string_bit_array_tests() -> List(Test) {
           <<codepoint:utf32_codepoint>>
         })
       }),
+    "UTF-32 codepoint little-endian"
+      |> example(fn() {
+        assert_equal(<<13, 243, 1, 0>>, {
+          // 🌍
+          let codepoint = ffi.utf_codepoint(127_757)
+          <<codepoint:utf32_codepoint-little>>
+        })
+      }),
   ]
 }