Repository: mgeisler/textwrap (language: rust)

Issue:
Automatic hyphenation
There is a `hyphenation` crate available on crates.io which provides hyphenation patterns for many languages. This can be used to automatically hyphenate the text, thus providing better wrapping.


Discussion / hints:

Fix patch:
diff --git a/Cargo.toml b/Cargo.toml
index bd4d26e5..5836afd1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,3 +11,4 @@ license = "MIT"
 
 [dependencies]
 unicode-width = "0.1.3"
+hyphenation = "0.6.1"
diff --git a/README.md b/README.md
index 5268ed7d..50174eee 100644
--- a/README.md
+++ b/README.md
@@ -51,7 +51,14 @@ results.
 ## Examples
 
 The library comes with a small example program that shows how a fixed
-example string is wrapped at different widths:
+example string is wrapped at different widths. The string is
+
+> Memory safety without garbage collection. Concurrency without data
+> races. Zero-cost abstractions.
+
+When run, the string is wrapped at all widths between 15 and 60
+columns:
+
 ```shell
 $ cargo run --example layout
 .--- Width: 15 ---.
@@ -61,26 +68,31 @@ $ cargo run --example layout
 | Concurrency     |
 | without data    |
 | races. Zero-    |
-| cost            |
-| abstractions.   |
+| cost abstrac-   |
+| tions.          |
 .--- Width: 16 ----.
 | Memory safety    |
 | without garbage  |
-| collection.      |
-| Concurrency      |
-| without data     |
-| races. Zero-cost |
-| abstractions.    |
+| collection. Con- |
+| currency without |
+| data races. Ze-  |
+| ro-cost abstrac- |
+| tions.           |
 # ...
-.---------------- Width: 41 ----------------.
-| Memory safety without garbage collection. |
-| Concurrency without data races. Zero-cost |
-| abstractions.                             |
+.-------------------- Width: 49 --------------------.
+| Memory safety without garbage collection. Concur- |
+| rency without data races. Zero-cost abstractions. |
 .---------------------- Width: 53 ----------------------.
 | Memory safety without garbage collection. Concurrency |
 | without data races. Zero-cost abstractions.           |
+.------------------------- Width: 59 -------------------------.
+| Memory safety without garbage collection. Concurrency with- |
+| out data races. Zero-cost abstractions.                     |
 ```
 
+Notice how words are split at hyphens (such a s "zero-cost") but also
+how words are hyphenated using automatic/machine hyphenation.
+
 ## Changelog
 
 * **Version 0.2.0, December 28 2016:** Introduced `Wrapper` struct.
diff --git a/examples/layout.rs b/examples/layout.rs
index 44bd24ca..5ddcb88e 100644
--- a/examples/layout.rs
+++ b/examples/layout.rs
@@ -1,16 +1,22 @@
+extern crate hyphenation;
 extern crate textwrap;
 
-use textwrap::wrap;
+use hyphenation::Language;
+use textwrap::Wrapper;
 
 fn main() {
+    let corpus = hyphenation::load(Language::English_US).unwrap();
     let example = "
 Memory safety without garbage collection.
 Concurrency without data races.
 Zero-cost abstractions.
 ";
     let mut prev_lines = vec![];
+    let mut wrapper = Wrapper::new(0);
+    wrapper.corpus = Some(&corpus);
     for width in 15..60 {
-        let lines = wrap(example, width);
+        wrapper.width = width;
+        let lines = wrapper.wrap(example);
         if lines != prev_lines {
             let title = format!(" Width: {} ", width);
             println!(".{:-^1$}.", title, width + 2);
diff --git a/src/lib.rs b/src/lib.rs
index 08a6c70a..139f370c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,19 +1,25 @@
 //! `textwrap` provides functions for word wrapping and filling text.
 
 extern crate unicode_width;
+extern crate hyphenation;
 
-use std::iter;
 use unicode_width::UnicodeWidthStr;
+use hyphenation::Hyphenation;
+use hyphenation::Corpus;
 
 /// A Wrapper holds settings for wrapping text.
-pub struct Wrapper {
-    width: usize,
+pub struct Wrapper<'a> {
+    pub width: usize,
+    pub corpus: Option<&'a Corpus>,
 }
 
-impl Wrapper {
+impl<'a> Wrapper<'a> {
     /// Create a new Wrapper for wrapping at the specified width.
-    pub fn new(width: usize) -> Wrapper {
-        Wrapper { width: width }
+    pub fn new(width: usize) -> Wrapper<'a> {
+        Wrapper::<'a> {
+            width: width,
+            corpus: None,
+        }
     }
 
     /// Fill a line of text at `self.width` characters. Strings are
@@ -54,28 +60,34 @@ impl Wrapper {
     /// ```
     pub fn wrap(&self, s: &str) -> Vec<String> {
         let mut result = Vec::new();
-        let mut line = Vec::new();
+        let mut line = String::new();
         let mut line_width = 0;
 
         for mut word in s.split_whitespace() {
             while !word.is_empty() {
-                let splits = split_word(&word);
-                let (smallest, longest) = splits[0];
-                let min_width = smallest.width();
+                let splits = self.split_word(&word);
+                let (smallest, hyphen, longest) = splits[0];
+                let min_width = smallest.width() + hyphen.len();
 
                 // Add a new line if even the smallest split doesn't
                 // fit.
-                if !line.is_empty() && line_width + line.len() + min_width > self.width {
-                    result.push(line.join(" "));
-                    line = Vec::new();
+                if !line.is_empty() && line_width + 1 + min_width > self.width {
+                    result.push(line);
+                    line = String::new();
                     line_width = 0;
                 }
 
+                let space = if line_width > 0 { 1 } else { 0 };
+
                 // Find a split that fits on the current line.
-                for &(head, tail) in splits.iter().rev() {
-                    if line_width + line.len() + head.width() <= self.width {
-                        line.push(head);
-                        line_width += head.width();
+                for &(head, hyphen, tail) in splits.iter().rev() {
+                    if line_width + space + head.width() + hyphen.len() <= self.width {
+                        if line_width > 0 {
+                            line.push(' ');
+                        }
+                        line.push_str(head);
+                        line.push_str(hyphen);
+                        line_width += space + head.width() + hyphen.len();
                         word = tail;
                         break;
                     }
@@ -84,15 +96,49 @@ impl Wrapper {
                 // If nothing got added, we forcibly add the smallest
                 // split and continue with the longest tail.
                 if line_width == 0 {
-                    result.push(String::from(smallest));
+                    result.push(String::from(smallest) + hyphen);
                     line_width = 0;
                     word = longest;
                 }
             }
         }
         if !line.is_empty() {
-            result.push(line.join(" "));
+            result.push(line);
+        }
+        return result;
+    }
+
+    /// Split word into all possible parts (head, tail). Word must be
+    /// non-empty. The returned vector will always be non-empty.
+    fn split_word<'b>(&self, word: &'b str) -> Vec<(&'b str, &'b str, &'b str)> {
+        let mut result = Vec::new();
+
+        // Split on hyphens or use the language corpus.
+        match self.corpus {
+            None => {
+                // Split on hyphens, smallest split first.
+                for (n, _) in word.match_indices('-') {
+                    let (head, tail) = word.split_at(n + 1);
+                    result.push((head, "", tail));
+                }
+            }
+            Some(corpus) => {
+                // Find splits based on language corpus. This includes
+                // the splits that would have been found above.
+                for n in word.opportunities(corpus) {
+                    let (head, tail) = word.split_at(n);
+                    let mut hyphen = "-";
+                    if head.as_bytes()[head.len() - 1] == b'-' {
+                        hyphen = "";
+                    }
+                    result.push((head, hyphen, tail));
+                }
+            }
         }
+
+        // Finally option is no split at all.
+        result.push((word, "", ""));
+
         return result;
     }
 }
@@ -110,8 +156,9 @@ impl Wrapper {
 ///            "Memory safety\nwithout garbage\ncollection.");
 /// ```
 ///
-/// This function creates a Wrapper on the fly. If you need to wrap
-/// many strings, it can be more efficient to create a single Wrapper
+/// This function creates a Wrapper on the fly with default settings.
+/// If you need to set a language corpus for automatic hyphenation, or
+/// need to fill many strings, then it is suggested to create Wrapper
 /// and call its [`fill` method](struct.Wrapper.html#method.fill).
 pub fn fill(s: &str, width: usize) -> String {
     wrap(s, width).join("\n")
@@ -133,23 +180,14 @@ pub fn fill(s: &str, width: usize) -> String {
 ///                 "data races."]);
 /// ```
 ///
-/// This function creates a Wrapper on the fly. If you need to wrap
-/// many strings, it can be more efficient to create a single Wrapper
+/// This function creates a Wrapper on the fly with default settings.
+/// If you need to set a language corpus for automatic hyphenation, or
+/// need to wrap many strings, then it is suggested to create Wrapper
 /// and call its [`wrap` method](struct.Wrapper.html#method.wrap).
 pub fn wrap(s: &str, width: usize) -> Vec<String> {
     Wrapper::new(width).wrap(s)
 }
 
-/// Split word into all possible parts (head, tail). Word must be
-/// non-empty. The returned vector will always be non-empty.
-fn split_word(word: &str) -> Vec<(&str, &str)> {
-    let hyphens = word.match_indices('-');
-    let word_end = iter::once((word.len() - 1, ""));
-    return hyphens.chain(word_end)
-        .map(|(n, _)| word.split_at(n + 1))
-        .collect();
-}
-
 /// Add prefix to each non-empty line.
 ///
 /// ```
@@ -243,6 +281,9 @@ pub fn dedent(s: &str) -> String {
 
 #[cfg(test)]
 mod tests {
+    extern crate hyphenation;
+
+    use hyphenation::Language;
     use super::*;
 
     /// Add newlines. Ensures that the final line in the vector also
@@ -313,6 +354,29 @@ mod tests {
         assert_eq!(wrap("foobar-baz", 5), vec!["foobar-", "baz"]);
     }
 
+    #[test]
+    fn auto_hyphenation() {
+        let corpus = hyphenation::load(Language::English_US).unwrap();
+        let mut wrapper = Wrapper::new(10);
+        assert_eq!(wrapper.wrap("Internationalization"),
+                   vec!["Internationalization"]);
+
+        wrapper.corpus = Some(&corpus);
+        assert_eq!(wrapper.wrap("Internationalization"),
+                   vec!["Interna-", "tionaliza-", "tion"]);
+    }
+
+    #[test]
+    fn auto_hyphenation_with_hyphen() {
+        let corpus = hyphenation::load(Language::English_US).unwrap();
+        let mut wrapper = Wrapper::new(8);
+        assert_eq!(wrapper.wrap("over-caffinated"), vec!["over-", "caffinated"]);
+
+        wrapper.corpus = Some(&corpus);
+        assert_eq!(wrapper.wrap("over-caffinated"),
+                   vec!["over-", "caffi-", "nated"]);
+    }
+
     #[test]
     fn test_fill() {
         assert_eq!(fill("foo bar baz", 10), "foo bar\nbaz");


Test patch: