Repository: swc-project/swc (rust)

Issue:
Source maps do not seem to handle multi-byte characters properly
### Describe the bug

I'm trying to switch to swc for our emits completely in Deno, but I ran into this bug. The mapping of source maps has the width of multi-byte characters too long.

For exmaple:

![image](https://user-images.githubusercontent.com/1609021/177885924-db9b537d-902e-40df-815a-647bdfa9795f.png)

...is mapped to two characters:

![image](https://user-images.githubusercontent.com/1609021/177885941-31a20c33-9dbd-4bf4-aa4c-292f749631bc.png)

Or this comment:

![image](https://user-images.githubusercontent.com/1609021/177886004-4dd582cb-8586-4ef5-afae-38215e2e0e71.png)

Mapped to:

![image](https://user-images.githubusercontent.com/1609021/177886020-951831d9-d602-4b84-9105-34c0877b9f3d.png)


### Input code

```typescript
// This entire interface should be completely ignored by the coverage tool.
export interface Complex {
  // These comments should be ignored.
  foo: string;

  // But this is a stub, so this isn't really documentation.
  bar: string;

  // Really all these are doing is padding the line count.
  baz: string;
}

// Lets add some wide characters to ensure that the absolute byte offsets are
// being matched properly.
//
// 패딩에 대한 더 많은 문자.
function dependency(
  foo: string,
  bar: string,
  baz: string,
): Complex {
  return {
    foo,
    bar,
    baz,
  };
}

// Again just more wide characters for padding.
//
// 良い対策のためにいくつかのユニコード文字を投げる。
export function complex(
  foo: string,
  bar: string,
  baz: string,
): Complex {
  return dependency(
    foo,
    bar,
    baz,
  );
}

// And yet again for good measure.
// 更多用於填充的字元。
export function unused(
  foo: string,
  bar: string,
  baz: string,
): Complex {
  return complex(
    foo,
    bar,
    baz,
  );
}

// Using a non-ascii name again to ensure that the byte offsets match up
// correctly.
export const π = Math.PI;

// And same applies for this one, this one is unused and will show up in
// lacking coverage.
export function ƒ(): number {
  return (
    0
  );
}

// This arrow function should also show up as uncovered.
console.log("%s", () => 1);

// Make sure emojis work properly
console.log("📣❓");
```


### Config

```json
{
  "$schema": "https://json.schemastore.org/swcrc",
  "jsc": {
    "parser": {
      "syntax": "typescript"
    },
    "target": "es2020",
    "loose": false,
    "externalHelpers": false
  },
  "sourceMaps": true
}
```


### Playground link

_No response_

### Expected behavior

Should be mapped to correct length.

### Actual behavior

https://evanw.github.io/source-map-visualization/#MTAyMwAvLyBMZXRzIGFkZCBzb21lIHdpZGUgY2hhcmFjdGVycyB0byBlbnN1cmUgdGhhdCB0aGUgYWJzb2x1dGUgYnl0ZSBvZmZzZXRzIGFyZQovLyBiZWluZyBtYXRjaGVkIHByb3Blcmx5LgovLwovLyDtjKjrlKnsl5Ag64yA7ZWcIOuNlCDrp47snYAg66y47J6QLgpmdW5jdGlvbiBkZXBlbmRlbmN5KGZvbywgYmFyLCBiYXopIHsKICAgIHJldHVybiB7CiAgICAgICAgZm9vLAogICAgICAgIGJhciwKICAgICAgICBiYXoKICAgIH07Cn0KLy8gQWdhaW4ganVzdCBtb3JlIHdpZGUgY2hhcmFjdGVycyBmb3IgcGFkZGluZy4KLy8KLy8g6Imv44GE5a++562W44Gu44Gf44KB44Gr44GE44GP44Gk44GL44Gu44Om44OL44Kz44O844OJ5paH5a2X44KS5oqV44GS44KL44CCCmV4cG9ydCBmdW5jdGlvbiBjb21wbGV4KGZvbywgYmFyLCBiYXopIHsKICAgIHJldHVybiBkZXBlbmRlbmN5KGZvbywgYmFyLCBiYXopOwp9Ci8vIEFuZCB5ZXQgYWdhaW4gZm9yIGdvb2QgbWVhc3VyZS4KLy8g5pu05aSa55So5pa85aGr5YWF55qE5a2X5YWD44CCCmV4cG9ydCBmdW5jdGlvbiB1bnVzZWQoZm9vLCBiYXIsIGJheikgewogICAgcmV0dXJuIGNvbXBsZXgoZm9vLCBiYXIsIGJheik7Cn0KLy8gVXNpbmcgYSBub24tYXNjaWkgbmFtZSBhZ2FpbiB0byBlbnN1cmUgdGhhdCB0aGUgYnl0ZSBvZmZzZXRzIG1hdGNoIHVwCi8vIGNvcnJlY3RseS4KZXhwb3J0IGNvbnN0IM+AID0gTWF0aC5QSTsKLy8gQW5kIHNhbWUgYXBwbGllcyBmb3IgdGhpcyBvbmUsIHRoaXMgb25lIGlzIHVudXNlZCBhbmQgd2lsbCBzaG93IHVwIGluCi8vIGxhY2tpbmcgY292ZXJhZ2UuCmV4cG9ydCBmdW5jdGlvbiDGkigpIHsKICAgIHJldHVybiAwOwp9Ci8vIFRoaXMgYXJyb3cgZnVuY3Rpb24gc2hvdWxkIGFsc28gc2hvdyB1cCBhcyB1bmNvdmVyZWQuCmNvbnNvbGUubG9nKCIlcyIsICgpPT4xKTsKLy8gTWFrZSBzdXJlIGVtb2ppcyB3b3JrIHByb3Blcmx5CmNvbnNvbGUubG9nKCLwn5Oj4p2TIik7CgoKLy8jIHNvdXJjZU1hcHBpbmdVUkw9ZmlsZS5qcy5tYXAyMzU3AHsidmVyc2lvbiI6Mywic291cmNlcyI6WyJmaWxlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQVlBLDZFQUE2RTtBQUM3RSwwQkFBMEI7QUFDMUIsRUFBRTtBQUNGLHNDQUFrQjtBQUNFLFNBQVgsVUFBVSxDQUNqQixHQUFXLEVBQ1gsR0FBVyxFQUNYLEdBQVcsRUFDRjtJQUNULE9BQU87UUFDTCxHQUFHO1FBQ0gsR0FBRztRQUNILEdBQUc7S0FDSixDQUFDO0NBQ0g7QUFFRCwrQ0FBK0M7QUFDL0MsRUFBRTtBQUNGLDhFQUE0QjtBQUNzQixPQUEzQyxTQUFTLE9BQU8sQ0FDckIsR0FBVyxFQUNYLEdBQVcsRUFDWCxHQUFXLEVBQ0Y7SUFDVCxPQUFPLFVBQVUsQ0FDZixHQUFHLEVBQ0gsR0FBRyxFQUNILEdBQUcsQ0FDSixDQUFDO0NBQ0g7QUFFRCxrQ0FBa0M7QUFDbEMsaUNBQWE7QUFDTyxPQUFiLFNBQVMsTUFBTSxDQUNwQixHQUFXLEVBQ1gsR0FBVyxFQUNYLEdBQVcsRUFDRjtJQUNULE9BQU8sT0FBTyxDQUNaLEdBQUcsRUFDSCxHQUFHLEVBQ0gsR0FBRyxDQUNKLENBQUM7Q0FDSDtBQUVELHdFQUF3RTtBQUN4RSxhQUFhO0FBQ2IsT0FBTyxNQUFNLEVBQUMsR0FBSSxJQUFJLENBQUMsRUFBRSxDQUFDO0FBRXpCLHdFQUF1RTtBQUN4RSxvQkFBb0I7QUFDcEIsT0FBTyxTQUFTLEVBQUMsR0FBWTtJQUMxQixPQUNDLENBQUMsQ0FDRDtDQUNIO0FBRUQsd0RBQXdEO0FBQ3hELE9BQU8sQ0FBQyxHQUFHLENBQUMsSUFBSSxFQUFFLElBQU0sQ0FBQyxDQUFDLENBQUM7QUFFM0IsaUNBQWlDO0FBQ2pDLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBSSxDQUFDLENBQUMiLCJmaWxlIjoiZmlsZS5qcyIsInNvdXJjZXNDb250ZW50IjpbIi8vIFRoaXMgZW50aXJlIGludGVyZmFjZSBzaG91bGQgYmUgY29tcGxldGVseSBpZ25vcmVkIGJ5IHRoZSBjb3ZlcmFnZSB0b29sLlxuZXhwb3J0IGludGVyZmFjZSBDb21wbGV4IHtcbiAgLy8gVGhlc2UgY29tbWVudHMgc2hvdWxkIGJlIGlnbm9yZWQuXG4gIGZvbzogc3RyaW5nO1xuXG4gIC8vIEJ1dCB0aGlzIGlzIGEgc3R1Yiwgc28gdGhpcyBpc24ndCByZWFsbHkgZG9jdW1lbnRhdGlvbi5cbiAgYmFyOiBzdHJpbmc7XG5cbiAgLy8gUmVhbGx5IGFsbCB0aGVzZSBhcmUgZG9pbmcgaXMgcGFkZGluZyB0aGUgbGluZSBjb3VudC5cbiAgYmF6OiBzdHJpbmc7XG59XG5cbi8vIExldHMgYWRkIHNvbWUgd2lkZSBjaGFyYWN0ZXJzIHRvIGVuc3VyZSB0aGF0IHRoZSBhYnNvbHV0ZSBieXRlIG9mZnNldHMgYXJlXG4vLyBiZWluZyBtYXRjaGVkIHByb3Blcmx5LlxuLy9cbi8vIO2MqOuUqeyXkCDrjIDtlZwg642UIOunjuydgCDrrLjsnpAuXG5mdW5jdGlvbiBkZXBlbmRlbmN5KFxuICBmb286IHN0cmluZyxcbiAgYmFyOiBzdHJpbmcsXG4gIGJhejogc3RyaW5nLFxuKTogQ29tcGxleCB7XG4gIHJldHVybiB7XG4gICAgZm9vLFxuICAgIGJhcixcbiAgICBiYXosXG4gIH07XG59XG5cbi8vIEFnYWluIGp1c3QgbW9yZSB3aWRlIGNoYXJhY3RlcnMgZm9yIHBhZGRpbmcuXG4vL1xuLy8g6Imv44GE5a++562W44Gu44Gf44KB44Gr44GE44GP44Gk44GL44Gu44Om44OL44Kz44O844OJ5paH5a2X44KS5oqV44GS44KL44CCXG5leHBvcnQgZnVuY3Rpb24gY29tcGxleChcbiAgZm9vOiBzdHJpbmcsXG4gIGJhcjogc3RyaW5nLFxuICBiYXo6IHN0cmluZyxcbik6IENvbXBsZXgge1xuICByZXR1cm4gZGVwZW5kZW5jeShcbiAgICBmb28sXG4gICAgYmFyLFxuICAgIGJheixcbiAgKTtcbn1cblxuLy8gQW5kIHlldCBhZ2FpbiBmb3IgZ29vZCBtZWFzdXJlLlxuLy8g5pu05aSa55So5pa85aGr5YWF55qE5a2X5YWD44CCXG5leHBvcnQgZnVuY3Rpb24gdW51c2VkKFxuICBmb286IHN0cmluZyxcbiAgYmFyOiBzdHJpbmcsXG4gIGJhejogc3RyaW5nLFxuKTogQ29tcGxleCB7XG4gIHJldHVybiBjb21wbGV4KFxuICAgIGZvbyxcbiAgICBiYXIsXG4gICAgYmF6LFxuICApO1xufVxuXG4vLyBVc2luZyBhIG5vbi1hc2NpaSBuYW1lIGFnYWluIHRvIGVuc3VyZSB0aGF0IHRoZSBieXRlIG9mZnNldHMgbWF0Y2ggdXBcbi8vIGNvcnJlY3RseS5cbmV4cG9ydCBjb25zdCDPgCA9IE1hdGguUEk7XG5cbi8vIEFuZCBzYW1lIGFwcGxpZXMgZm9yIHRoaXMgb25lLCB0aGlzIG9uZSBpcyB1bnVzZWQgYW5kIHdpbGwgc2hvdyB1cCBpblxuLy8gbGFja2luZyBjb3ZlcmFnZS5cbmV4cG9ydCBmdW5jdGlvbiDGkigpOiBudW1iZXIge1xuICByZXR1cm4gKFxuICAgIDBcbiAgKTtcbn1cblxuLy8gVGhpcyBhcnJvdyBmdW5jdGlvbiBzaG91bGQgYWxzbyBzaG93IHVwIGFzIHVuY292ZXJlZC5cbmNvbnNvbGUubG9nKFwiJXNcIiwgKCkgPT4gMSk7XG5cbi8vIE1ha2Ugc3VyZSBlbW9qaXMgd29yayBwcm9wZXJseVxuY29uc29sZS5sb2coXCLwn5Oj4p2TXCIpO1xuIl19

### Version

swc/core: 1.2.210

### Additional context

_No response_

Pull request:
fix(source-map): better handling of multi-byte characters
**Description:**

Uses character lengths instead of byte lengths in some places.

**Related issue (if exists):**

Closes #5145

Patch:
diff --git a/crates/swc_common/src/source_map.rs b/crates/swc_common/src/source_map.rs
index 8fbd93dbfd..4b5b9f43b9 100644
--- a/crates/swc_common/src/source_map.rs
+++ b/crates/swc_common/src/source_map.rs
@@ -1157,8 +1157,6 @@ impl SourceMap {
 
         let mut cur_file: Option<Lrc<SourceFile>> = None;
 
-        let mut ch_start = 0;
-        let mut line_ch_start = 0;
         let mut prev_dst_line = u32::MAX;
 
         for (pos, lc) in mappings.iter() {
@@ -1193,8 +1191,6 @@ impl SourceMap {
                     }
 
                     cur_file = Some(f.clone());
-                    ch_start = 0;
-                    line_ch_start = 0;
                     &f
                 }
             };
@@ -1228,9 +1224,8 @@ impl SourceMap {
                 pos,
                 linebpos,
             );
-            let chpos = pos.to_u32() - self.calc_extra_bytes(f, &mut ch_start, pos);
-            let linechpos =
-                linebpos.to_u32() - self.calc_extra_bytes(f, &mut line_ch_start, linebpos);
+            let chpos = pos.to_u32() - self.calc_extra_bytes(f, &mut 0, pos);
+            let linechpos = linebpos.to_u32() - self.calc_extra_bytes(f, &mut 0, linebpos);
 
             let mut col = max(chpos, linechpos) - min(chpos, linechpos);
 
diff --git a/crates/swc_ecma_codegen/src/text_writer/basic_impl.rs b/crates/swc_ecma_codegen/src/text_writer/basic_impl.rs
index 7fe5f87a5d..a16f92af38 100644
--- a/crates/swc_ecma_codegen/src/text_writer/basic_impl.rs
+++ b/crates/swc_ecma_codegen/src/text_writer/basic_impl.rs
@@ -1,4 +1,4 @@
-use std::io::{self, Write};
+use std::io::Write;
 
 use rustc_hash::FxHashSet;
 use swc_common::{sync::Lrc, BytePos, LineCol, SourceMap, Span};
@@ -44,29 +44,26 @@ impl<'a, W: Write> JsWriter<'a, W> {
         }
     }
 
-    fn write_indent_string(&mut self) -> io::Result<usize> {
-        const INDENT: &[u8] = b"    ";
+    fn write_indent_string(&mut self) -> Result {
+        const INDENT: &str = "    ";
 
-        let mut cnt = 0;
         for _ in 0..self.indent {
-            cnt += self.raw_write(INDENT)?;
+            self.raw_write(INDENT)?;
         }
 
-        Ok(cnt)
+        Ok(())
     }
 
-    fn raw_write(&mut self, data: &[u8]) -> io::Result<usize> {
-        let written = self.wr.write(data)?;
-        self.line_pos += written;
-        Ok(written)
+    fn raw_write(&mut self, data: &str) -> Result {
+        self.wr.write_all(data.as_bytes())?;
+        self.line_pos += data.chars().count();
+        Ok(())
     }
 
-    fn write(&mut self, span: Option<Span>, data: &str) -> io::Result<usize> {
-        let mut cnt = 0;
-
+    fn write(&mut self, span: Option<Span>, data: &str) -> Result {
         if !data.is_empty() {
             if self.line_start {
-                cnt += self.write_indent_string()?;
+                self.write_indent_string()?;
                 self.line_start = false;
 
                 if let Some(pending) = self.pending_srcmap.take() {
@@ -80,7 +77,7 @@ impl<'a, W: Write> JsWriter<'a, W> {
                 }
             }
 
-            cnt += self.raw_write(data.as_bytes())?;
+            self.raw_write(data)?;
 
             if let Some(span) = span {
                 if !span.is_dummy() {
@@ -89,7 +86,7 @@ impl<'a, W: Write> JsWriter<'a, W> {
             }
         }
 
-        Ok(cnt)
+        Ok(())
     }
 
     fn srcmap(&mut self, byte_pos: BytePos) {
@@ -156,7 +153,7 @@ impl<'a, W: Write> WriteJs for JsWriter<'a, W> {
     fn write_line(&mut self) -> Result {
         let pending = self.pending_srcmap.take();
         if !self.line_start {
-            self.raw_write(self.new_line.as_bytes())?;
+            self.raw_write(self.new_line)?;
             self.line_count += 1;
             self.line_pos = 0;
             self.line_start = true;
@@ -180,7 +177,8 @@ impl<'a, W: Write> WriteJs for JsWriter<'a, W> {
             let line_start_of_s = compute_line_starts(s);
             if line_start_of_s.len() > 1 {
                 self.line_count = self.line_count + line_start_of_s.len() - 1;
-                self.line_pos = s.len() - line_start_of_s.last().cloned().unwrap_or(0);
+                let last_line_byte_index = line_start_of_s.last().cloned().unwrap_or(0);
+                self.line_pos = s[last_line_byte_index..].chars().count();
             }
 
             if !span.is_dummy() {
@@ -197,7 +195,8 @@ impl<'a, W: Write> WriteJs for JsWriter<'a, W> {
             let line_start_of_s = compute_line_starts(s);
             if line_start_of_s.len() > 1 {
                 self.line_count = self.line_count + line_start_of_s.len() - 1;
-                self.line_pos = s.len() - line_start_of_s.last().cloned().unwrap_or(0);
+                let last_line_byte_index = line_start_of_s.last().cloned().unwrap_or(0);
+                self.line_pos = s[last_line_byte_index..].chars().count();
             }
         }
         Ok(())
@@ -214,7 +213,8 @@ impl<'a, W: Write> WriteJs for JsWriter<'a, W> {
             let line_start_of_s = compute_line_starts(s);
             if line_start_of_s.len() > 1 {
                 self.line_count = self.line_count + line_start_of_s.len() - 1;
-                self.line_pos = s.len() - line_start_of_s.last().cloned().unwrap_or(0);
+                let last_line_byte_index = line_start_of_s.last().cloned().unwrap_or(0);
+                self.line_pos = s[last_line_byte_index..].chars().count();
             }
 
             if !span.is_dummy() {


Test patch:
diff --git a/crates/swc/tests/fixture/issues-4xxx/4120/1/output/index.map b/crates/swc/tests/fixture/issues-4xxx/4120/1/output/index.map
index a693872f56..c9334db9ce 100644
--- a/crates/swc/tests/fixture/issues-4xxx/4120/1/output/index.map
+++ b/crates/swc/tests/fixture/issues-4xxx/4120/1/output/index.map
@@ -1,5 +1,5 @@
 {
-  "mappings": "AAUA,cAAe,CAAEA,CAAC,CATd,wDAAsF,CAStEC,CAAC,CALjB,yGAA6J,CAK1I,AAAC",
+  "mappings": "AAUA,cAAe,CAAEA,CAAC,CATd,yCAAsF,CAStEC,CAAC,CALjB,kFAA6J,CAK1I,AAAC",
   "names": [
     "a",
     "b"
diff --git a/crates/swc/tests/source_map.rs b/crates/swc/tests/source_map.rs
index 28b9e05ce7..188b905c18 100644
--- a/crates/swc/tests/source_map.rs
+++ b/crates/swc/tests/source_map.rs
@@ -243,7 +243,7 @@ fn issue_4112() {
                 r#""use strict";
 
             export { default as Selection } from "./selection";
-            
+
             export { default as RichTextarea } from "./richTextarea";
             "#
                 .to_string(),
@@ -268,7 +268,7 @@ fn issue_4112() {
                 r#""use strict";
 
             import { React, window } from "easy";
-            
+
             window.assign({
               React
             });
@@ -311,9 +311,13 @@ fn should_work_with_emit_source_map_columns() {
         let fm = cm.new_source_file(
             swc_common::FileName::Real("./app.js".into()),
             r#"import { createElement } from "react";
-      
-  createElement('div', null, {});"#
-                .to_string(),
+
+  createElement('div', null, {});
+  /* 패딩에 대 */ /* 한 더 많은 문자. */ "테스트" + "테스트";
+  function ƒ() {
+  }
+"#
+            .to_string(),
         );
         let result = c.process_js_file(
             fm.clone(),
@@ -335,6 +339,8 @@ fn should_work_with_emit_source_map_columns() {
             Ok(result) => {
                 assert!(result.map.is_some());
                 let map = result.map.unwrap();
+
+                // lookup createElement(...) function call
                 let source_map = sourcemap::SourceMap::from_slice(map.as_bytes())
                     .expect("failed to deserialize sourcemap");
                 let token = source_map
@@ -344,6 +350,24 @@ fn should_work_with_emit_source_map_columns() {
                 assert_eq!(token.get_dst_col(), 14);
                 assert_eq!(token.get_src_line(), 2);
                 assert_eq!(token.get_src_col(), 16);
+
+                // lookup second string literal
+                let token = source_map
+                    .lookup_token(2, 37)
+                    .expect("failed to find token");
+                assert_eq!(token.get_dst_line(), 2);
+                assert_eq!(token.get_dst_col(), 37);
+                assert_eq!(token.get_src_line(), 3);
+                assert_eq!(token.get_src_col(), 39);
+
+                // lookup paren after multi-byte identifier
+                let token = source_map
+                    .lookup_token(3, 10)
+                    .expect("failed to find token");
+                assert_eq!(token.get_dst_line(), 3);
+                assert_eq!(token.get_dst_col(), 10);
+                assert_eq!(token.get_src_line(), 4);
+                assert_eq!(token.get_src_col(), 12);
             }
             Err(err) => {
                 panic!("Error: {:#?}", err);