Repository: bytecodealliance/wasm-tools
Pull request #1235 (2023-10-06 00:28:40)

Add options to control the functions used to allocate memory
This PR adds 2 options to control how realloc and the stack allocation is implemented.  The options if supplied will make these functions use `memory.grow` rather than the `cabi_realloc` (if supplied) of the component.

My first PR here, and pretty new to Rust, so feedback welcomed!

Discussed over in zulip.

Thanks,


Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 2417781d..9329e3ba 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -649,12 +649,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flagset"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499"
-dependencies = [
- "serde",
-]
+checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779"
 
 [[package]]
 name = "flate2"
@@ -1717,6 +1714,7 @@ dependencies = [
  "petgraph",
  "pretty_assertions",
  "serde",
+ "serde_derive",
  "serde_yaml",
  "smallvec",
  "wasm-encoder 0.33.2",
@@ -1753,6 +1751,7 @@ dependencies = [
  "clap 4.3.22",
  "indexmap 2.0.0",
  "serde",
+ "serde_derive",
  "serde_json",
  "spdx",
  "wasm-encoder 0.33.2",
@@ -1823,6 +1822,7 @@ dependencies = [
  "libfuzzer-sys",
  "rand",
  "serde",
+ "serde_derive",
  "wasm-encoder 0.33.2",
  "wasmparser 0.113.3",
  "wasmprinter 0.2.68",
@@ -1848,6 +1848,7 @@ dependencies = [
  "regex",
  "rustc-demangle",
  "serde",
+ "serde_derive",
  "serde_json",
  "tempfile",
  "termcolor",
@@ -2360,6 +2361,7 @@ dependencies = [
  "log",
  "pretty_assertions",
  "serde",
+ "serde_derive",
  "serde_json",
  "wasm-encoder 0.33.2",
  "wasm-metadata",
@@ -2397,13 +2399,12 @@ dependencies = [
  "indexmap 2.0.0",
  "log",
  "pretty_assertions",
- "pulldown-cmark",
  "rayon",
  "semver",
  "serde",
+ "serde_derive",
  "serde_json",
  "unicode-xid",
- "url",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index fdb861df..60689d9a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,7 +37,8 @@ log = "0.4.17"
 num_cpus = "1.13"
 rand = { version = "0.8.4", features = ["small_rng"] }
 rayon = "1.3"
-serde = { version = "1.0.166", features = ["derive"] }
+serde = "1.0.166"
+serde_derive = "1.0.166"
 serde_json = { version = "1" }
 wasmtime = { version = "12.0.0", default-features = false, features = ['cranelift', 'component-model'] }
 url = "2.0.0"
@@ -77,6 +78,7 @@ wasmprinter = { workspace = true }
 # Dependencies of `smith`
 arbitrary = { workspace = true, optional = true }
 serde = { workspace = true, optional = true }
+serde_derive = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }
 wasm-smith = { workspace = true, features = ["_internal_cli"], optional = true }
 
@@ -156,7 +158,7 @@ default = [
 validate = ['dep:wasmparser', 'rayon']
 print = []
 parse = []
-smith = ['wasm-smith', 'arbitrary', 'serde', 'serde_json']
+smith = ['wasm-smith', 'arbitrary', 'serde', 'serde_derive', 'serde_json']
 shrink = ['wasm-shrink', 'is_executable']
 mutate = ['wasm-mutate']
 dump = ['dep:wasmparser']
diff --git a/crates/wasm-compose/Cargo.toml b/crates/wasm-compose/Cargo.toml
index 7a9f2ece..39c39165 100644
--- a/crates/wasm-compose/Cargo.toml
+++ b/crates/wasm-compose/Cargo.toml
@@ -17,6 +17,7 @@ wasmparser = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
 anyhow = { workspace = true }
 serde = { workspace = true }
+serde_derive = { workspace = true }
 petgraph = "0.6.2"
 log = { workspace = true }
 serde_yaml = "0.9.22"
diff --git a/crates/wasm-compose/src/config.rs b/crates/wasm-compose/src/config.rs
index 82029a7b..84f1df67 100644
--- a/crates/wasm-compose/src/config.rs
+++ b/crates/wasm-compose/src/config.rs
@@ -2,7 +2,7 @@
 
 use anyhow::{Context, Result};
 use indexmap::IndexMap;
-use serde::Deserialize;
+use serde_derive::Deserialize;
 use std::{
     fs,
     path::{Path, PathBuf},
diff --git a/crates/wasm-metadata/Cargo.toml b/crates/wasm-metadata/Cargo.toml
index 2350c122..485ab0f4 100644
--- a/crates/wasm-metadata/Cargo.toml
+++ b/crates/wasm-metadata/Cargo.toml
@@ -13,6 +13,7 @@ wasmparser = { workspace = true }
 wasm-encoder = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
 serde = { workspace = true }
+serde_derive = { workspace = true }
 serde_json = { version = "1" }
 spdx = "0.10.1"
 
diff --git a/crates/wasm-metadata/src/lib.rs b/crates/wasm-metadata/src/lib.rs
index 47429d02..53f8ec08 100644
--- a/crates/wasm-metadata/src/lib.rs
+++ b/crates/wasm-metadata/src/lib.rs
@@ -1,6 +1,6 @@
 use anyhow::Result;
 use indexmap::{map::Entry, IndexMap};
-use serde::{Deserialize, Serialize};
+use serde_derive::{Deserialize, Serialize};
 use spdx::Expression;
 use std::borrow::Cow;
 use std::fmt;
diff --git a/crates/wasm-smith/Cargo.toml b/crates/wasm-smith/Cargo.toml
index 5bf9bf56..dce3ea21 100644
--- a/crates/wasm-smith/Cargo.toml
+++ b/crates/wasm-smith/Cargo.toml
@@ -21,6 +21,7 @@ flagset = "0.4"
 indexmap = { workspace = true }
 leb128 = { workspace = true }
 serde = { workspace = true, optional = true }
+serde_derive = { workspace = true, optional = true }
 wasm-encoder = { workspace = true }
 wasmparser = { workspace = true }
 
@@ -34,4 +35,4 @@ wat = { path = "../wat" }
 libfuzzer-sys = { workspace = true }
 
 [features]
-_internal_cli = ["serde", "flagset/serde"]
+_internal_cli = ["serde", "serde_derive"]
diff --git a/crates/wasm-smith/src/core.rs b/crates/wasm-smith/src/core.rs
index b8629232..9fe15faa 100644
--- a/crates/wasm-smith/src/core.rs
+++ b/crates/wasm-smith/src/core.rs
@@ -1721,7 +1721,7 @@ flags! {
     /// Enumerate the categories of instructions defined in the [WebAssembly
     /// specification](https://webassembly.github.io/spec/core/syntax/instructions.html).
     #[allow(missing_docs)]
-    #[cfg_attr(feature = "_internal_cli", derive(serde::Deserialize))]
+    #[cfg_attr(feature = "_internal_cli", derive(serde_derive::Deserialize))]
     pub enum InstructionKind: u16 {
         Numeric,
         Vector,
diff --git a/crates/wit-component/Cargo.toml b/crates/wit-component/Cargo.toml
index 948c6384..6fbc3656 100644
--- a/crates/wit-component/Cargo.toml
+++ b/crates/wit-component/Cargo.toml
@@ -24,6 +24,7 @@ indexmap = { workspace = true }
 wast = { workspace = true }
 wat = { workspace = true }
 serde = { workspace = true }
+serde_derive = { workspace = true }
 serde_json = { workspace = true }
 
 [dev-dependencies]
diff --git a/crates/wit-component/src/encoding.rs b/crates/wit-component/src/encoding.rs
index 598aa85a..6a596087 100644
--- a/crates/wit-component/src/encoding.rs
+++ b/crates/wit-component/src/encoding.rs
@@ -1892,6 +1892,8 @@ pub struct ComponentEncoder {
     validate: bool,
     main_module_exports: IndexSet<WorldKey>,
     adapters: IndexMap<String, Adapter>,
+
+    realloc_via_memory_grow: bool,
 }
 
 impl ComponentEncoder {
@@ -2030,6 +2032,15 @@ impl ComponentEncoder {
         Ok(self)
     }
 
+    /// True if the realloc and stack alloction should use ememory.grow
+    /// The default is to use the main module realloc
+    /// Can be useful if cabi_realloc cannot be called before the host
+    /// runtime is initialised.
+    pub fn realloc_via_memory_grow(mut self, value: bool) -> Self {
+        self.realloc_via_memory_grow = value;
+        self
+    }
+
     /// Encode the component and return the bytes.
     pub fn encode(&self) -> Result<Vec<u8>> {
         if self.module.is_empty() {
diff --git a/crates/wit-component/src/encoding/docs.rs b/crates/wit-component/src/encoding/docs.rs
index 99ddd82a..accdbc2d 100644
--- a/crates/wit-component/src/encoding/docs.rs
+++ b/crates/wit-component/src/encoding/docs.rs
@@ -1,6 +1,6 @@
 use anyhow::{bail, Result};
 use indexmap::IndexMap;
-use serde::{Deserialize, Serialize};
+use serde_derive::{Deserialize, Serialize};
 use wasm_encoder::{CustomSection, Encode};
 use wit_parser::{Docs, InterfaceId, PackageId, Resolve, TypeId, WorldId, WorldItem, WorldKey};
 
diff --git a/crates/wit-component/src/encoding/world.rs b/crates/wit-component/src/encoding/world.rs
index 79c9c52e..47b90878 100644
--- a/crates/wit-component/src/encoding/world.rs
+++ b/crates/wit-component/src/encoding/world.rs
@@ -137,8 +137,16 @@ impl<'a> ComponentWorld<'a> {
                 );
 
                 Cow::Owned(
-                    crate::gc::run(wasm, &required, self.info.realloc)
-                        .context("failed to reduce input adapter module to its minimal size")?,
+                    crate::gc::run(
+                        wasm,
+                        &required,
+                        if self.encoder.realloc_via_memory_grow {
+                            None
+                        } else {
+                            self.info.realloc
+                        },
+                    )
+                    .context("failed to reduce input adapter module to its minimal size")?,
                 )
             };
             let info = validate_adapter_module(
diff --git a/crates/wit-parser/Cargo.toml b/crates/wit-parser/Cargo.toml
index 2e04b0f3..10ca1453 100644
--- a/crates/wit-parser/Cargo.toml
+++ b/crates/wit-parser/Cargo.toml
@@ -16,12 +16,11 @@ Tooling for parsing `*.wit` files and working with their contents.
 id-arena = "2"
 anyhow = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
-pulldown-cmark = { version = "0.9.3", default-features = false }
 unicode-xid = "0.2.2"
 log = { workspace = true }
-url = { workspace = true }
 semver = { workspace = true }
-serde.workspace = true
+serde = { workspace = true }
+serde_derive = { workspace = true }
 serde_json = "1.0.105"
 
 [dev-dependencies]
diff --git a/crates/wit-parser/src/ast.rs b/crates/wit-parser/src/ast.rs
index aca7a464..4f386c26 100644
--- a/crates/wit-parser/src/ast.rs
+++ b/crates/wit-parser/src/ast.rs
@@ -1199,11 +1199,7 @@ impl SourceMap {
     /// interfaces and worlds defined together. Note that each file has its own
     /// personal namespace, however, for top-level `use` and such.
     pub fn push(&mut self, path: &Path, contents: impl Into<String>) {
-        let mut contents = contents.into();
-        if path.extension().and_then(|s| s.to_str()) == Some("md") {
-            log::debug!("automatically unwrapping markdown container");
-            contents = unwrap_md(&contents);
-        }
+        let contents = contents.into();
         let new_offset = self.offset + u32::try_from(contents.len()).unwrap();
         self.sources.push(Source {
             offset: self.offset,
@@ -1211,43 +1207,6 @@ impl SourceMap {
             contents,
         });
         self.offset = new_offset;
-
-        fn unwrap_md(contents: &str) -> String {
-            use pulldown_cmark::{CodeBlockKind, CowStr, Event, Options, Parser, Tag};
-
-            let mut wit = String::new();
-            let mut last_pos = 0;
-            let mut in_wit_code_block = false;
-            Parser::new_ext(contents, Options::empty())
-                .into_offset_iter()
-                .for_each(|(event, range)| match (event, range) {
-                    (
-                        Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(CowStr::Borrowed(
-                            "wit",
-                        )))),
-                        _,
-                    ) => {
-                        in_wit_code_block = true;
-                    }
-                    (Event::Text(text), range) if in_wit_code_block => {
-                        // Ensure that offsets are correct by inserting newlines to
-                        // cover the Markdown content outside of wit code blocks.
-                        for _ in contents[last_pos..range.start].lines() {
-                            wit.push('\n');
-                        }
-                        wit.push_str(&text);
-                        last_pos = range.end;
-                    }
-                    (
-                        Event::End(Tag::CodeBlock(CodeBlockKind::Fenced(CowStr::Borrowed("wit")))),
-                        _,
-                    ) => {
-                        in_wit_code_block = false;
-                    }
-                    _ => {}
-                });
-            wit
-        }
     }
 
     /// Parses the files added to this source map into an [`UnresolvedPackage`].
diff --git a/crates/wit-parser/src/lib.rs b/crates/wit-parser/src/lib.rs
index 696a8c19..100b8e49 100644
--- a/crates/wit-parser/src/lib.rs
+++ b/crates/wit-parser/src/lib.rs
@@ -2,7 +2,7 @@ use anyhow::{Context, Result};
 use id_arena::{Arena, Id};
 use indexmap::IndexMap;
 use semver::Version;
-use serde::Serialize;
+use serde_derive::Serialize;
 use std::borrow::Cow;
 use std::fmt;
 use std::path::Path;
diff --git a/crates/wit-parser/src/resolve.rs b/crates/wit-parser/src/resolve.rs
index 2c15a201..f98fa4d6 100644
--- a/crates/wit-parser/src/resolve.rs
+++ b/crates/wit-parser/src/resolve.rs
@@ -9,7 +9,7 @@ use crate::{
 use anyhow::{anyhow, bail, Context, Result};
 use id_arena::{Arena, Id};
 use indexmap::{IndexMap, IndexSet};
-use serde::Serialize;
+use serde_derive::Serialize;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::mem;
 use std::path::{Path, PathBuf};
diff --git a/crates/wit-parser/src/serde_.rs b/crates/wit-parser/src/serde_.rs
index c31eda84..f7af8799 100644
--- a/crates/wit-parser/src/serde_.rs
+++ b/crates/wit-parser/src/serde_.rs
@@ -99,7 +99,7 @@ where
     seq.end()
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize)]
+#[derive(Debug, Clone, PartialEq, serde_derive::Serialize)]
 struct Param {
     #[serde(skip_serializing_if = "String::is_empty")]
     pub name: String,
diff --git a/src/bin/wasm-tools/component.rs b/src/bin/wasm-tools/component.rs
index 10addc20..0ba7aa3b 100644
--- a/src/bin/wasm-tools/component.rs
+++ b/src/bin/wasm-tools/component.rs
@@ -110,6 +110,10 @@ pub struct NewOpts {
     /// Print the output in the WebAssembly text format instead of binary.
     #[clap(long, short = 't')]
     wat: bool,
+
+    /// Use memory.grow to realloc memory and stack allocation.
+    #[clap(long)]
+    realloc_via_memory_grow: bool,
 }
 
 impl NewOpts {
@@ -128,6 +132,8 @@ impl NewOpts {
             encoder = encoder.adapter(name, wasm)?;
         }
 
+        encoder = encoder.realloc_via_memory_grow(self.realloc_via_memory_grow);
+
         let bytes = encoder
             .encode()
             .context("failed to encode a component from module")?;
diff --git a/src/bin/wasm-tools/smith.rs b/src/bin/wasm-tools/smith.rs
index fac3546e..700383c0 100644
--- a/src/bin/wasm-tools/smith.rs
+++ b/src/bin/wasm-tools/smith.rs
@@ -79,7 +79,7 @@ pub struct Opts {
     general: wasm_tools::GeneralOpts,
 }
 
-#[derive(Default, Debug, Parser, Clone, serde::Deserialize)]
+#[derive(Default, Debug, Parser, Clone, serde_derive::Deserialize)]
 #[serde(rename_all = "kebab-case")]
 struct Config {
     #[clap(long = "min-types")]


Test patch:
diff --git a/crates/wit-parser/tests/ui/embedded.wit.md b/crates/wit-parser/tests/ui/embedded.wit.md
deleted file mode 100644
index 0a1e8082..00000000
--- a/crates/wit-parser/tests/ui/embedded.wit.md
+++ /dev/null
@@ -1,34 +0,0 @@
-# A Markdown file!
-
-containing stuff, and also some code blocks, wit and other.
-
-```wit
-package foo:foo;
-
-interface foo {
-```
-
-```wit
-x: func();
-```
-
-Intervening content, including a non-wit codeblock:
-```js
-function func() {}
-```
-
-```wit
-y: func();
-```
-
-## A new section
-
-In which, another wit code block!
-
-```wit
-z: func();
-```
-
-```wit
-}
-```
diff --git a/crates/wit-parser/tests/ui/embedded.wit.md.json b/crates/wit-parser/tests/ui/embedded.wit.md.json
deleted file mode 100644
index 83a2a44a..00000000
--- a/crates/wit-parser/tests/ui/embedded.wit.md.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-  "worlds": [],
-  "interfaces": [
-    {
-      "name": "foo",
-      "types": {},
-      "functions": {
-        "x": {
-          "name": "x",
-          "kind": "freestanding",
-          "params": [],
-          "results": []
-        },
-        "y": {
-          "name": "y",
-          "kind": "freestanding",
-          "params": [],
-          "results": []
-        },
-        "z": {
-          "name": "z",
-          "kind": "freestanding",
-          "params": [],
-          "results": []
-        }
-      },
-      "package": 0
-    }
-  ],
-  "types": [],
-  "packages": [
-    {
-      "name": "foo:foo",
-      "interfaces": {
-        "foo": 0
-      },
-      "worlds": {}
-    }
-  ]
-}
\ No newline at end of file
diff --git a/crates/wit-parser/tests/ui/parse-fail/invalid-md.md b/crates/wit-parser/tests/ui/parse-fail/invalid-md.md
deleted file mode 100644
index 9464aaa2..00000000
--- a/crates/wit-parser/tests/ui/parse-fail/invalid-md.md
+++ /dev/null
@@ -1,7 +0,0 @@
-// parse-fail
-
-hello
-
-```wit
-type foo = bar
-```
diff --git a/crates/wit-parser/tests/ui/parse-fail/invalid-md.wit.result b/crates/wit-parser/tests/ui/parse-fail/invalid-md.wit.result
deleted file mode 100644
index ed4f7e1c..00000000
--- a/crates/wit-parser/tests/ui/parse-fail/invalid-md.wit.result
+++ /dev/null
@@ -1,5 +0,0 @@
-expected `world`, `interface` or `use`, found keyword `type`
-     --> tests/ui/parse-fail/invalid-md.md:6:1
-      |
-    6 | type foo = bar
-      | ^---
\ No newline at end of file