Repository: alvinwan/TexSoup

Issue:
edge cases
- [x] \ can denote escape (special characters: { , : ; ) or be used after e.g.\
- [ ] \right. \right[ \right( \right|

again, noted by/thanks to @chewisinho for bringing up


Discussion:

Patch:
diff --git a/TexSoup/__init__.py b/TexSoup/__init__.py
index c34f8b7..ec90adb 100644
--- a/TexSoup/__init__.py
+++ b/TexSoup/__init__.py
@@ -8,8 +8,6 @@ Main file, containing most commonly used elements of TexSoup
 @site: alvinwan.com
 """
 
-import itertools
-import _io
 from TexSoup.tex import *
 
 
diff --git a/TexSoup/data.py b/TexSoup/data.py
index 93926d5..6ccb199 100644
--- a/TexSoup/data.py
+++ b/TexSoup/data.py
@@ -212,7 +212,7 @@ class TexExpr(object):
         """Further breaks down all tokens for a particular expression into
         words and other expressions."""
         for content in self.contents:
-            if isinstance(str, content):
+            if isinstance(content, str):
                 for word in content.split():
                     yield word
             yield content
@@ -229,10 +229,10 @@ class TexEnv(TexExpr):
     environment arguments, whether optional or required, and (3) the
     environment's contents.
 
-    >>> t = TexEnv('tabular', ['0 & 0 & * \\\\', '1 & 1 & * \\\\'],
+    >>> t = TexEnv('tabular', ['\n0 & 0 & * \\\\\n1 & 1 & * \\\\\n'],
     ...     [RArg('c | c c')])
     >>> t
-    TexEnv('tabular', ['0 & 0 & * \\\\', '1 & 1 & * \\\\'], [RArg('c | c c')])
+    TexEnv('tabular', ['\n0 & 0 & * \\\\\n1 & 1 & * \\\\\n'], [RArg('c | c c')])
     >>> print(t)
     \begin{tabular}{c | c c}
     0 & 0 & * \\
@@ -270,7 +270,7 @@ class TexEnv(TexExpr):
         if self.nobegin:
             return '%s%s%s' % (
                 self.name, contents, self.name)
-        return '\\begin{%s}%s\n%s\n\\end{%s}' % (
+        return '\\begin{%s}%s%s\\end{%s}' % (
             self.name, self.args, contents, self.name)
 
     def __repr__(self):
diff --git a/TexSoup/reader.py b/TexSoup/reader.py
index 2e9842f..5772cea 100644
--- a/TexSoup/reader.py
+++ b/TexSoup/reader.py
@@ -4,8 +4,7 @@ import TexSoup.data as data
 import functools
 import itertools
 
-__all__ = ['read_line', 'read_lines', 'tokenize_line', 'tokenize_lines',
-    'read_tex']
+__all__ = ['tokenize', 'read_tex']
 
 COMMAND_TOKENS = {'\\'}
 MATH_TOKENS = {'$'}
@@ -14,46 +13,20 @@ ARG_END_TOKENS = {arg.delims()[1] for arg in data.args}
 ARG_TOKENS = ARG_START_TOKENS | ARG_END_TOKENS
 ALL_TOKENS = COMMAND_TOKENS | ARG_TOKENS | MATH_TOKENS
 SKIP_ENVS = ('verbatim', 'equation', 'lstlisting')
+PUNCTUATION_COMMANDS = ('right', 'left')
 
 
-#######################
-# Convenience Methods #
-#######################
-
-
-def read_line(line):
-    r"""Read first expression from a single line
-
-    >>> read_line(r'\textbf{Do play \textit{nice}.}')
-    TexCmd('textbf', [RArg('Do play ', TexCmd('textit', [RArg('nice')]), '.')])
-    >>> print(read_line(r'\newcommand{solution}[1]{{\color{blue} #1}}'))
-    \newcommand{solution}[1]{{\color{blue} #1}}
-    """
-    return read_tex(Buffer(tokenize_line(line)))
-
-
-def read_lines(*lines):
-    r"""Read first expression from multiple lines
-
-    >>> print(read_lines(r'\begin{tabular}{c c}', '0 & 1 \\\\',
-    ...     '\end{tabular}'))
-    \begin{tabular}{c c}
-    0 & 1 \\
-    \end{tabular}
-    """
-    return read_tex(Buffer(itertools.chain(*tokenize_lines(lines))))
-
 #############
 # Tokenizer #
 #############
 
 
 @to_buffer
-def next_token(line):
+def next_token(text):
     r"""Returns the next possible token, advancing the iterator to the next
     position to start processing from.
 
-    :param (str, iterator, Buffer) line: LaTeX to process
+    :param (str, iterator, Buffer) text: LaTeX to process
     :return str: the token
 
     >>> b = Buffer(r'\textbf{Do play\textit{nice}.}   $$\min_w \|w\|_2^2$$')
@@ -71,38 +44,30 @@ def next_token(line):
     '$$\\min_w \\|w\\|_2^2$$'
     >>> next_token(b)
     """
-    while line.hasNext():
+    while text.hasNext():
         for name, f in tokenizers:
-            token = f(line)
+            token = f(text)
             if token is not None:
                 return token
 
 
 @to_buffer
-def tokenize_line(line):
-    r"""Generator for LaTeX tokens on a single line, ignoring comments.
+def tokenize(text):
+    r"""Generator for LaTeX tokens on text, ignoring comments.
 
-    :param (str, iterator, Buffer) line: LaTeX to process
+    :param (str, iterator, Buffer) text: LaTeX to process
 
-    >>> print(*tokenize_line(r'\textbf{Do play \textit{nice}.}'))
+    >>> print(*tokenize(r'\textbf{Do play \textit{nice}.}'))
     \ textbf { Do play  \ textit { nice } . }
-    >>> print(*tokenize_line(r'\begin{tabular} 0 & 1 \\ 2 & 0 \end{tabular}'))
+    >>> print(*tokenize(r'\begin{tabular} 0 & 1 \\ 2 & 0 \end{tabular}'))
     \ begin { tabular }  0 & 1 \\ 2 & 0  \ end { tabular }
-    >>> print(*tokenize_line(r'$$\min_x \|Xw-y\|_2^2$$'))
+    >>> print(*tokenize(r'$$\min_x \|Xw-y\|_2^2$$'))
     $$\min_x \|Xw-y\|_2^2$$
     """
-    token = next_token(line)
+    token = next_token(text)
     while token is not None:
         yield token
-        token = next_token(line)
-
-
-def tokenize_lines(lines):
-    """Generator for LaTeX tokens across multiple lines, ignoring comments.
-
-    :param list lines: list of strings or iterator over strings
-    """
-    return map(tokenize_line, lines)
+        token = next_token(text)
 
 
 ##########
@@ -123,52 +88,67 @@ def token(name):
     return wrap
 
 
+@token('punctuation_command')
+def tokenize_punctuation_command(text):
+    """Process command that augments or modifies punctuation.
+
+    This is important to the tokenization of a string, as opening or closing
+    punctuation is not supposed to match.
+
+    :param Buffer text: iterator over text, with current position
+    """
+    if text.peek() == '\\':
+        for string in PUNCTUATION_COMMANDS:
+            if text.peek((1, len(string) + 1)) == string:
+                return text.forward(len(string) + 3)
+
+
 @token('command')
-def tokenize_command(line):
+def tokenize_command(text):
     """Process command, but ignore line breaks. (double backslash)
 
-    :param Buffer line: iterator over line, with current position
+    :param Buffer text: iterator over line, with current position
     """
-    if line.peek() == '\\' and line.peek(1) not in ALL_TOKENS:
-        return next(line)
+    if text.peek() == '\\' and text.peek(1) not in ALL_TOKENS:
+        return next(text)
 
 
 @token('argument')
-def tokenize_argument(line):
+def tokenize_argument(text):
     """Process both optional and required arguments.
 
-    :param Buffer line: iterator over line, with current position
+    :param Buffer text: iterator over line, with current position
     """
     for delim in ARG_TOKENS:
-        if line.startswith(delim):
-            return line.forward(len(delim))
+        if text.startswith(delim):
+            return text.forward(len(delim))
 
 
 @token('math')
-def tokenize_math(line):
+def tokenize_math(text):
     r"""Prevents math from being tokenized.
 
-    :param Buffer line: iterator over line, with current position
+    :param Buffer text: iterator over line, with current position
 
     >>> b = Buffer('$$\min_x$$ \command')
     >>> tokenize_math(b)
     '$$\\min_x$$'
     """
     result = ''
-    if line.startswith('$'):
-        starter = '$$' if line.startswith('$$') else '$'
-        result += line.forward(len(starter))
-        while line.hasNext() and line.peek((0, len(starter))) != starter:
-            result += next(line)
-        if not line.startswith(starter):
+    if text.startswith('$'):
+        starter = '$$' if text.startswith('$$') else '$'
+        result += text.forward(len(starter))
+        while text.hasNext() and text.peek((0, len(starter))) != starter:
+            result += next(text)
+        if not text.startswith(starter):
             raise EOFError('Expecting %s. Instead got %s' % (
-                starter, line.peek((0, 5))))
-        result += line.forward(len(starter))
+                starter, text.peek((0, 5))))
+        result += text.forward(len(starter))
         return result
 
 
 @token('string')
-def tokenize_string(line, delimiters=ALL_TOKENS):
+def tokenize_string(text, delimiters=ALL_TOKENS):
     r"""Process a string of text
 
     :param Buffer line: iterator over line, with current position
@@ -184,15 +164,15 @@ def tokenize_string(line, delimiters=ALL_TOKENS):
     0 & 1 \\
     """
     result = ''
-    for c in line:
-        if c == '\\' and line.peek() in delimiters:
-            c += next(line)
+    for c in text:
+        if c == '\\' and text.peek() in delimiters:
+            c += next(text)
         elif c in delimiters:  # assumes all tokens are single characters
-            line.backward(1)
+            text.backward(1)
             return result
         result += c
-        if line.peek((0, 2)) == '\\\\':
-            result += line.forward(2)
+        if text.peek((0, 2)) == '\\\\':
+            result += text.forward(2)
     return result
 
 
@@ -213,15 +193,16 @@ def read_tex(src):
     if c == '\\':
         if src.peek().startswith('item '):
             mode, expr = 'command', TexCmd('item', (),
-                ' '.join(next(src).split(' ')[1:]))
+                ' '.join(next(src).split(' ')[1:]).strip())
         elif src.peek() == 'begin':
             mode, expr = next(src), TexEnv(Arg.parse(src.forward(3)).value)
         else:
-            mode, candidate = 'command', next(src)
-            if ' ' in candidate:
-                tokens = candidate.split(' ')
-                expr = TexCmd(tokens[0], (), ' '.join(tokens[1:]))
-            else:
+            mode, candidate, expr = 'command', next(src), None
+            for i, c in enumerate(candidate):
+                if c.isspace():
+                    expr = TexCmd(candidate[:i], (), candidate[i+1:])
+                    break
+            if not expr:
                 expr = TexCmd(candidate)
         while src.peek() in ARG_START_TOKENS:
             expr.args.append(read_tex(src))
@@ -230,6 +211,8 @@ def read_tex(src):
         if src.startswith('$'):
             expr.add_contents(read_tex(src))
         return expr
+    if c.startswith('\\'):
+        return TexCmd(c[1:])
     if c in ARG_START_TOKENS:
         return read_arg(src, c)
     return c
diff --git a/TexSoup/tex.py b/TexSoup/tex.py
index 555e7b7..a384906 100644
--- a/TexSoup/tex.py
+++ b/TexSoup/tex.py
@@ -11,8 +11,10 @@ def read(tex):
     :return TexEnv: the global environment
     """
     if isinstance(tex, str):
-        tex = tex.strip().splitlines()
-    buf, children = Buffer(itertools.chain(*tokenize_lines(tex))), []
+        tex = tex.strip()
+    else:
+        tex = itertools.chain(*tex)
+    buf, children = Buffer(tokenize(tex)), []
     while buf.hasNext():
         content = read_tex(buf)
         if content is not None:
diff --git a/TexSoup/utils.py b/TexSoup/utils.py
index 6f2af40..acbd865 100644
--- a/TexSoup/utils.py
+++ b/TexSoup/utils.py
@@ -15,12 +15,22 @@ import functools
 # Decorators #
 ##############
 
+
 def to_buffer(f, i=0):
     """
     Decorator converting all strings and iterators/iterables into
     Buffers.
 
     :param int i: index of iterator argument. Used only if not a kwarg.
+
+    >>> f = to_buffer(lambda x: x[:])
+    >>> f('asdf')
+    'asdf'
+    >>> g = to_buffer(lambda x: x)
+    >>> g('').hasNext()
+    False
+    >>> next(g('asdf'))
+    'a'
     """
     @functools.wraps(f)
     def wrap(*args, **kwargs):
@@ -39,6 +49,7 @@ def to_buffer(f, i=0):
 # Generalized Utilities #
 #########################
 
+
 class Buffer:
     """Converts string or iterable into a navigable iterator of strings
 


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 7d99360..de08b65 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -95,7 +95,7 @@ def test_command_name_parse():
     with_linebreak_not_arg = TexSoup(r"""\Question
 (10 points)""")
     assert with_linebreak_not_arg.Question is not None
-    assert with_linebreak_not_arg.Question.extra == ''
+    assert with_linebreak_not_arg.Question.extra == '(10 points)'
 
     with_space_with_arg = TexSoup(r"""\section {hula}""")
     assert with_space_with_arg.section.string == 'hula'
@@ -141,14 +141,16 @@ def test_ignore_environment():
     \min_x \|Ax - b\|_2^2 + \lambda \|x\|_2^2
     \end{verbatim}
     $$\min_x \|Ax - b\|_2^2 + \lambda \|x\|_1^2$$
+    $$[0,1)$$
     """)
     verbatim = list(list(soup.children)[1].contents)[0]
-    assert len(list(soup.contents)) == 3, 'Special environments not recognized.'
+    assert len(list(soup.contents)) == 4, 'Special environments not recognized.'
     assert str(list(soup.children)[0]) == \
-           '\\begin{equation}\n\min_x \|Ax - b\|_2^2\n\\end{equation}'
-    assert verbatim.startswith('    '), 'Whitespace not preserved.'
+           '\\begin{equation}\min_x \|Ax - b\|_2^2\\end{equation}'
+    assert verbatim.startswith('\n    '), 'Whitespace not preserved.'
     assert str(list(soup.children)[2]) == \
         '$$\min_x \|Ax - b\|_2^2 + \lambda \|x\|_1^2$$'
+    assert str(list(soup.children)[3]) == '$$[0,1)$$'
 
 
 def test_inline_math():
@@ -169,11 +171,58 @@ def test_escaped_characters():
     """
     soup = TexSoup("""
     \begin{itemize}
-    \item Ice cream costs \$4-\$5 around here.
+    \item Ice cream costs \$4-\$5 around here. \}\]\{\[
     \end{itemize}""")
+    assert str(soup.item) == r'\item Ice cream costs \$4-\$5 around here. ' \
+                             r'\}\]\{\['
     assert '\\$4-\\$5' in str(soup), 'Escaped characters not properly rendered.'
 
 
+##############
+# FORMATTING #
+##############
+
+
+def test_basic_whitespace():
+    """Tests that basic text maintains whitespace."""
+    soup = TexSoup("""
+    Here is some text
+    with a line break
+    and awko      taco spacing
+    """)
+    assert len(str(soup).split('\n')) == 3, 'Line breaks not persisted.'
+
+
+def test_whitespace_in_command():
+    """Tests that whitespace in commands are maintained."""
+    soup = TexSoup(r"""
+    \begin{article}
+    \title {This title contains    a space}
+    \section {This title contains
+    line break}
+    \end{article}
+    """)
+    assert '    ' in soup.article.title.string
+    assert '\n' in soup.article.section.string
+
+
+def test_math_environment_whitespace():
+    """Tests that math environments are untouched."""
+    soup = TexSoup("""$$\lambda
+    \Sigma$$ But don't mind me \$3.00""")
+    children, contents = list(soup.children), list(soup.contents)
+    assert '\n' in str(children[0]), 'Whitesapce not preserved in math env.'
+    assert len(children) == 1 and children[0].name == '$$', 'Math env wrong'
+    assert '\$' in contents[1], 'Dollar sign not escaped!'
+
+
+def test_punctuation_command_structure():
+    """Tests that commands for punctuation work."""
+    soup = TexSoup(r"""\right. \right[ \right( \right|""")
+    assert len(list(soup.contents)) == 4
+    assert len(list(soup.children)) == 4
+
+
 ##########
 # ERRORS #
 ##########
@@ -186,6 +235,7 @@ def test_unclosed_environments():
 
 
 def test_unclosed_math_environments():
+    """Tests that unclosed math environment results in error."""
     with pytest.raises(EOFError):
         TexSoup(r"""$$\min_x \|Xw-y\|_2^2""")