Repository: tyx-editor/TyX
Pull request #24: feat: open typst document
Author: login: Myriad-Dreamin
Created: 2025-05-09T21:22:04Z | Merged: 2025-05-11T20:34:57Z

Closes #20.

This PR only implements typst to tiptap (prose mirror json) importer but not a patcher (patch tiptap changes to the origin typst document). Blocked by #22.

First, I dumped the currently used tiptap schema as [tyx.schema.json](https://github.com/Myriad-Dreamin/TyX/blob/509df7e0fcdc3d70fc33ebfc24cbd7c0c2f4b3a8/crates/tyx-tiptap-schema/assets/tyx.schema.json), then I generate rust structs using the schema.

Then, I compiles the typst document into tiptap json, the conversion path:

```typ
Typst Source Code
  -- typst compilation (HTML export) --> XML
  -- typlite (tinymist) --> Markdown
  -- tyx-tiptap-typst (this PR) --> tiptap json
```

An importer means we can edit any typst documents in TyX but cannot apply changes back to the origin document.

## Example Conversion

Input:

```typ
= Hello, World!
This is a typst document.
```

Output:

```json
{
 "type": "doc",
 "content": [
  {
   "type": "heading",
   "content": [
    {
     "type": "text",
     "content": [
      {
       "type": "plain",
       "content": "Hello, World!"
      }
     ]
    }
   ],
   "dir": null,
   "level": 2
  },
  {
   "type": "paragraph",
   "content": [
    {
     "type": "text",
     "content": [
      {
       "type": "plain",
       "content": "This is a typst document."
      }
     ]
    }
   ],
   "dir": null
  }
 ]
}
```





Linked issue #20: Add a Typst to TyX compiler and importer
Suggested by @Myriad-Dreamin in the discussion in #17.

Diff:
diff --git a/Cargo.lock b/Cargo.lock
index fae4cb1..d0bd77f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6341,6 +6341,8 @@ dependencies = [
  "tauri-plugin-process",
  "tauri-plugin-shell",
  "tauri-plugin-updater",
+ "tinymist-project",
+ "tyx-tiptap-typst",
  "url",
 ]
 
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index db3f468..bcc2ee6 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -20,12 +20,14 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 tauri-build = { version = "2", features = [] }
 
 [dependencies]
+serde.workspace = true
+serde_json.workspace = true
 tauri = { version = "2", features = [] }
 tauri-plugin-shell = "2"
 tauri-plugin-dialog = "2"
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
 tauri-plugin-process = "2"
+tinymist-project = { workspace = true, features = ["lsp"] }
+tyx-tiptap-typst.workspace = true
 url = "2.5.4"
 
 [target."cfg(not(any(target_os = \"android\", target_os = \"ios\")))".dependencies]
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index 5c9246b..ec9f9c7 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -1,7 +1,8 @@
 use std::{
     fs::File,
-    io::{Read, Write},
+    io::Write,
     path::{Path, PathBuf},
+    sync::{Arc, LazyLock},
 };
 
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -10,7 +11,21 @@ use tauri::{Emitter, Manager};
 
 use tauri_plugin_dialog::DialogExt;
 use tauri_plugin_shell::ShellExt;
+use tinymist_project::{CompileOnceArgs, EntryReader, LspUniverse, TaskInputs, WorldProvider};
+
+/// A global shared instance to process typst in a workspace.
+// todo: load compile arguments
+static VERSE: LazyLock<LspUniverse> = LazyLock::new(|| {
+    CompileOnceArgs {
+        root: Some(PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("..")),
+        input: Some("../main.typ".to_string()),
+        ..CompileOnceArgs::default()
+    }
+    .resolve()
+    .unwrap()
+});
 
+//
 #[tauri::command]
 fn save(filename: &str, content: &str) {
     if let Ok(mut f) = File::create(filename) {
@@ -39,12 +54,35 @@ fn saveas(handle: tauri::AppHandle) {
 }
 
 fn openfile(handle: &tauri::AppHandle, path: &Path) {
-    let h = handle.clone();
-    if let Ok(mut f) = File::open(path) {
-        let mut buffer = String::new();
-        f.read_to_string(&mut buffer).unwrap();
-        h.emit("open", (path.to_str().unwrap(), buffer)).unwrap();
-    }
+    let buffer = if path.extension().is_some_and(|ext| ext == "typ") {
+        let Ok(entry) = VERSE.entry_state().try_select_path_in_workspace(path) else {
+            eprintln!("not selected path {:?}", VERSE.entry_state());
+            return;
+        };
+        let world = VERSE.snapshot_with(Some(TaskInputs {
+            entry,
+            ..TaskInputs::default()
+        }));
+        if world.main_id().is_none() {
+            eprintln!("no main id");
+            return;
+        }
+
+        let Some(mut doc) = tyx_tiptap_typst::convert(Arc::new(world)) else {
+            return;
+        };
+        doc.filename = path.to_str().unwrap().to_string();
+        // todo: we serialize it here.
+        serde_json::to_string(&doc).unwrap()
+    } else {
+        // tyx
+        std::fs::read_to_string(path).unwrap()
+    };
+    // eprintln!("open tyx file: {buffer:?}");
+
+    handle
+        .emit("open", (path.to_str().unwrap(), buffer))
+        .unwrap();
 }
 
 #[tauri::command]
@@ -53,7 +91,7 @@ fn open(handle: tauri::AppHandle) {
     handle
         .dialog()
         .file()
-        .add_filter("tyx", &["tyx"])
+        .add_filter("tyx", &["tyx", "typ"])
         .pick_file(move |f| {
             if let Some(f) = f {
                 if let Some(path) = f.as_path() {


Reviews:
user: login: noamzaks
body: Thank you for the quick work! This is incredible.

I left a few notes (mostly for me) to further improve this
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-11T20:34:50Z

Review comments:
diff_hunk: @@ -10,7 +11,21 @@ use tauri::{Emitter, Manager};
 
 use tauri_plugin_dialog::DialogExt;
 use tauri_plugin_shell::ShellExt;
+use tinymist_project::{CompileOnceArgs, EntryReader, LspUniverse, TaskInputs, WorldProvider};
+
+/// A global shared instance to process typst in a workspace.
+// todo: load compile arguments
+static VERSE: LazyLock<LspUniverse> = LazyLock::new(|| {
+    CompileOnceArgs {
+        root: Some(PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("..")),
+        input: Some("../main.typ".to_string()),
+        ..CompileOnceArgs::default()
path: src-tauri/src/lib.rs
user: login: noamzaks
body: We need to change this in the future - from my testing this limits the root directory to be the TyX directory for now, right?
created_at: 2025-05-11T20:31:57Z
author_association: MEMBER
diff_hunk: @@ -39,12 +54,35 @@ fn saveas(handle: tauri::AppHandle) {
 }
 
 fn openfile(handle: &tauri::AppHandle, path: &Path) {
-    let h = handle.clone();
-    if let Ok(mut f) = File::open(path) {
-        let mut buffer = String::new();
-        f.read_to_string(&mut buffer).unwrap();
-        h.emit("open", (path.to_str().unwrap(), buffer)).unwrap();
-    }
+    let buffer = if path.extension().is_some_and(|ext| ext == "typ") {
+        let Ok(entry) = VERSE.entry_state().try_select_path_in_workspace(path) else {
+            eprintln!("not selected path {:?}", VERSE.entry_state());
+            return;
+        };
+        let world = VERSE.snapshot_with(Some(TaskInputs {
+            entry,
+            ..TaskInputs::default()
+        }));
+        if world.main_id().is_none() {
+            eprintln!("no main id");
+            return;
+        }
+
+        let Some(mut doc) = tyx_tiptap_typst::convert(Arc::new(world)) else {
+            return;
+        };
+        doc.filename = path.to_str().unwrap().to_string();
+        // todo: we serialize it here.
+        serde_json::to_string(&doc).unwrap()
+    } else {
+        // tyx
+        std::fs::read_to_string(path).unwrap()
+    };
+    // eprintln!("open tyx file: {buffer:?}");
path: src-tauri/src/lib.rs
user: login: noamzaks
body: ```suggestion
```
created_at: 2025-05-11T20:32:07Z
author_association: MEMBER
diff_hunk: @@ -39,12 +54,35 @@ fn saveas(handle: tauri::AppHandle) {
 }
 
 fn openfile(handle: &tauri::AppHandle, path: &Path) {
-    let h = handle.clone();
-    if let Ok(mut f) = File::open(path) {
-        let mut buffer = String::new();
-        f.read_to_string(&mut buffer).unwrap();
-        h.emit("open", (path.to_str().unwrap(), buffer)).unwrap();
-    }
+    let buffer = if path.extension().is_some_and(|ext| ext == "typ") {
+        let Ok(entry) = VERSE.entry_state().try_select_path_in_workspace(path) else {
+            eprintln!("not selected path {:?}", VERSE.entry_state());
+            return;
+        };
+        let world = VERSE.snapshot_with(Some(TaskInputs {
+            entry,
+            ..TaskInputs::default()
+        }));
+        if world.main_id().is_none() {
+            eprintln!("no main id");
+            return;
+        }
+
+        let Some(mut doc) = tyx_tiptap_typst::convert(Arc::new(world)) else {
+            return;
+        };
+        doc.filename = path.to_str().unwrap().to_string();
path: src-tauri/src/lib.rs
user: login: noamzaks
body: This is unnecessary as it gets overridden in TypeScript in the onOpen function:

```typescript
openDocuments.push({ ...JSON.parse(content), filename })
```

On the other hand, we will need to change that function to rename the filename to end in `.tyx` and also not override the `.typ` file, perhaps something like `basename (Imported).tyx`
created_at: 2025-05-11T20:33:47Z
author_association: MEMBER

PR comments:
user: login: noamzaks
created_at: 2025-05-10T07:00:26Z
body: This is incredible.
I assume this isn't ready yet because there is no way to call the importer from the app - update me if you need any help.

A quick note - the marks are not a node, they're part of the text node:
```json
{
    "type": "paragraph",
    "attrs": { "textAlign": "", "dir": "ltr" },
    "content": [
        {
            "type": "text",
            "marks": [{ "type": "bold" }],
            "text": "Test"
        }
    ]
}
```

Also worth noting about the example conversion that the level should probably be 1 (guessing 2 is for h2 in HTML export, but in the editor/typst it should be level 1)

Also noting to self:
- Need to open an issue for all of the unhandled elements discovered in the schema - thank you!
- Need to add CI for these tests
author_association: MEMBER
user: login: Myriad-Dreamin
created_at: 2025-05-10T07:20:41Z
body: > This is incredible. I assume this isn't ready yet because there is no way to call the importer from the app - update me if you need any help.
> 
> A quick note - the marks are not a node, they're part of the text node:
> 
> ```json
> {
>     "type": "paragraph",
>     "attrs": { "textAlign": "", "dir": "ltr" },
>     "content": [
>         {
>             "type": "text",
>             "marks": [{ "type": "bold" }],
>             "text": "Test"
>         }
>     ]
> }
> ```

Yeah, I only converted them without considering correctness. It just looked almost correct. These obvious bugs will be fixed once we start to manage to render the importer result correctly. I lack of knowledge about format of tiptap format so I may be making some stupid mistakes. I did search on network but get little information about the tiptap's json. That's why I dumped the [tyx.schema.json](https://github.com/Myriad-Dreamin/TyX/blob/509df7e0fcdc3d70fc33ebfc24cbd7c0c2f4b3a8/crates/tyx-tiptap-schema/assets/tyx.schema.json) from memory for reference.

> Also noting to self:
> 
> * Need to open an issue for all of the unhandled elements discovered in the schema - thank you!

Will do it. Not about the unhandled elements, but I find some kind of bad smell of the scheme, currently the naming (type field) of nodes and marks are a bit inconsistent, and I'm not sure whether we can make them consistent on our hand. For example:
- `listItem`: `camelCase`
- `math-block`: `kebab-case`
- `blockquote`: some other random case.

> * Need to add CI for these tests

I might only aim at rendering a "Hello World" example document in this PR, is it okay? I only made some simple snapshot testing and you can run them by:

```rs
cargo install cargo-insta # install
cargo insta test -p tyx-tiptap-typst --accept # test
```

Some test cases, stored in `crates/tyx-tiptap-typst/src/fixtures/integration`, are borrowed from tinymist and may not cover all the cases for tyx.


author_association: CONTRIBUTOR
user: login: Myriad-Dreamin
created_at: 2025-05-10T07:22:42Z
body: 
> * `blockquote`: some other random case.

Might be merely a legal html tag name, but not a good english word.


author_association: CONTRIBUTOR
user: login: noamzaks
created_at: 2025-05-10T07:28:54Z
body: - You're right on the tag name. I'm changing math-block and math-inline to be mathBlock and mathInline.
	They're implemented as a custom extension to TipTap. Thanks!
- blockquote is a default TipTap extension so I'd rather not change that.
- A simple hello world is great for the scope of the PR.
- I will review the PR when you say it's ready
author_association: MEMBER
user: login: noamzaks
created_at: 2025-05-10T07:31:11Z
body: See https://github.com/tyx-editor/TyX/commit/3edda91400667e333764b9d42bbb01151f10466e for the camelCase renaming
author_association: MEMBER
user: login: Myriad-Dreamin
created_at: 2025-05-10T10:27:24Z
body: I opened a few sub PRs, each of which only focus one thing that could be merged without affecting the stability of the main branch: 
- #26
- #27
- #28
- #29
- The rest is in this PR, #24

> > Also noting to self:
> > 
> > * Need to open an issue for all of the unhandled elements discovered in the schema - thank you!
> 
> Will do it.

I opened a few issues that I found in this PR: 
- #30
- #31
- #32
- #33

author_association: CONTRIBUTOR
user: login: Myriad-Dreamin
created_at: 2025-05-11T20:22:32Z
body: It is ready and you can open the fixture files to test it.

## Base Document

![image](https://github.com/user-attachments/assets/28ebe886-e632-4deb-af60-5b4466d797c1)

## Link Marks

![image](https://github.com/user-attachments/assets/9b16f804-7abf-4631-ab33-346a957d0d43)




author_association: CONTRIBUTOR