//! This module defines the tokens of the shell language.

use self::Token::*;
use std::fmt;

/// The inner representation of a positional parameter.
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
pub enum Positional {
    /// $0
    Zero,
    /// $1
    One,
    /// $2
    Two,
    /// $3
    Three,
    /// $4
    Four,
    /// $5
    Five,
    /// $6
    Six,
    /// $7
    Seven,
    /// $8
    Eight,
    /// $9
    Nine,
}

impl Positional {
    /// Converts a `Positional` as a numeric representation
    pub fn as_num(&self) -> u8 {
        match *self {
            Positional::Zero => 0,
            Positional::One => 1,
            Positional::Two => 2,
            Positional::Three => 3,
            Positional::Four => 4,
            Positional::Five => 5,
            Positional::Six => 6,
            Positional::Seven => 7,
            Positional::Eight => 8,
            Positional::Nine => 9,
        }
    }

    /// Attempts to convert a number to a `Positional` representation
    pub fn from_num(num: u8) -> Option<Self> {
        match num {
            0 => Some(Positional::Zero),
            1 => Some(Positional::One),
            2 => Some(Positional::Two),
            3 => Some(Positional::Three),
            4 => Some(Positional::Four),
            5 => Some(Positional::Five),
            6 => Some(Positional::Six),
            7 => Some(Positional::Seven),
            8 => Some(Positional::Eight),
            9 => Some(Positional::Nine),
            _ => None,
        }
    }
}

impl Into<u8> for Positional {
    fn into(self) -> u8 {
        self.as_num()
    }
}

/// The representation of (context free) shell tokens.
#[derive(PartialEq, Eq, Debug, Clone)]
pub enum Token {
    /// \n
    Newline,

    /// (
    ParenOpen,
    /// )
    ParenClose,
    /// {
    CurlyOpen,
    /// }
    CurlyClose,
    /// [
    SquareOpen,
    /// ]
    SquareClose,

    /// !
    Bang,
    /// ~
    Tilde,
    /// \#
    Pound,
    /// *
    Star,
    /// ?
    Question,
    /// \\
    Backslash,
    /// %
    Percent,
    /// \-
    Dash,
    /// \=
    Equals,
    /// +
    Plus,
    /// :
    Colon,
    /// @
    At,
    /// ^
    Caret,
    /// /
    Slash,
    /// ,
    Comma,

    /// '
    SingleQuote,
    /// "
    DoubleQuote,
    /// `
    Backtick,

    /// ;
    Semi,
    /// &
    Amp,
    /// |
    Pipe,
    /// &&
    AndIf,
    /// ||
    OrIf,
    /// ;;
    DSemi,

    /// <
    Less,
    /// \>
    Great,
    /// <<
    DLess,
    /// \>>
    DGreat,
    /// \>&
    GreatAnd,
    /// <&
    LessAnd,
    /// <<-
    DLessDash,
    /// \>|
    Clobber,
    /// <>
    LessGreat,

    /// $
    Dollar,
    /// $0, $1, ..., $9
    ///
    /// Must be its own token to avoid lumping the positional parameter
    /// as a `Literal` if the parameter is concatenated to something.
    ParamPositional(Positional),

    /// Any string of whitespace characters NOT including a newline.
    Whitespace(String),

    /// Any literal delimited by whitespace.
    Literal(String),
    /// A `Literal` capable of being used as a variable or function name. According to the POSIX
    /// standard it should only contain alphanumerics or underscores, and does not start with a digit.
    Name(String),
}

impl fmt::Display for Token {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(fmt, "{}", self.as_str())
    }
}

impl Token {
    /// Returns if the token's length is zero.
    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    /// Returns the number of characters it took to recognize a token.
    pub fn len(&self) -> usize {
        self.as_str().len()
    }

    /// Indicates whether a word can be delimited by this token
    /// when the token is **not** quoted or escaped.
    pub fn is_word_delimiter(&self) -> bool {
        match *self {
            Newline | ParenOpen | ParenClose | Semi | Amp | Less | Great | Pipe | AndIf | OrIf
            | DSemi | DLess | DGreat | GreatAnd | LessAnd | DLessDash | Clobber | LessGreat
            | Whitespace(_) => true,

            Bang | Star | Question | Backslash | SingleQuote | DoubleQuote | Backtick | Percent
            | Dash | Equals | Plus | Colon | At | Caret | Slash | Comma | CurlyOpen
            | CurlyClose | SquareOpen | SquareClose | Dollar | Tilde | Pound | Name(_)
            | Literal(_) | ParamPositional(_) => false,
        }
    }

    /// Gets a representation of the token as a string slice.
    pub fn as_str(&self) -> &str {
        match *self {
            Newline => "\n",
            ParenOpen => "(",
            ParenClose => ")",
            CurlyOpen => "{",
            CurlyClose => "}",
            SquareOpen => "[",
            SquareClose => "]",
            Dollar => "$",
            Bang => "!",
            Semi => ";",
            Amp => "&",
            Less => "<",
            Great => ">",
            Pipe => "|",
            Tilde => "~",
            Pound => "#",
            Star => "*",
            Question => "?",
            Backslash => "\\",
            Percent => "%",
            Dash => "-",
            Equals => "=",
            Plus => "+",
            Colon => ":",
            At => "@",
            Caret => "^",
            Slash => "/",
            Comma => ",",
            SingleQuote => "\'",
            DoubleQuote => "\"",
            Backtick => "`",
            AndIf => "&&",
            OrIf => "||",
            DSemi => ";;",
            DLess => "<<",
            DGreat => ">>",
            GreatAnd => ">&",
            LessAnd => "<&",
            DLessDash => "<<-",
            Clobber => ">|",
            LessGreat => "<>",

            ParamPositional(Positional::Zero) => "$0",
            ParamPositional(Positional::One) => "$1",
            ParamPositional(Positional::Two) => "$2",
            ParamPositional(Positional::Three) => "$3",
            ParamPositional(Positional::Four) => "$4",
            ParamPositional(Positional::Five) => "$5",
            ParamPositional(Positional::Six) => "$6",
            ParamPositional(Positional::Seven) => "$7",
            ParamPositional(Positional::Eight) => "$8",
            ParamPositional(Positional::Nine) => "$9",

            Whitespace(ref s) | Name(ref s) | Literal(ref s) => s,
        }
    }
}


//! Defines an interfaces to receive parse data and construct ASTs.
//!
//! This allows the parser to remain agnostic of the required source
//! representation, and frees up the library user to substitute their own.
//! If one does not require a custom AST representation, this module offers
//! a reasonable default builder implementation.
//!
//! If a custom AST representation is required you will need to implement
//! the `Builder` trait for your AST. Otherwise you can provide the `DefaultBuilder`
//! struct to the parser if you wish to use the default AST implementation.

use crate::ast::{AndOr, DefaultArithmetic, DefaultParameter, RedirectOrCmdWord, RedirectOrEnvVar};

mod default_builder;
mod empty_builder;

pub use self::default_builder::*;
pub use self::empty_builder::EmptyBuilder;

/// An indicator to the builder of how complete commands are separated.
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum SeparatorKind {
    /// A semicolon appears between commands, normally indicating a sequence.
    Semi,
    /// An ampersand appears between commands, normally indicating an asyncronous job.
    Amp,
    /// A newline (and possibly a comment) appears at the end of a command before the next.
    Newline,
    /// The command was delimited by a token (e.g. a compound command delimiter) or
    /// the end of input, but is *not* followed by another sequential command.
    Other,
}

/// An indicator to the builder whether a `while` or `until` command was parsed.
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
pub enum LoopKind {
    /// A `while` command was parsed, normally indicating the loop's body should be run
    /// while the guard's exit status is successful.
    While,
    /// An `until` command was parsed, normally indicating the loop's body should be run
    /// until the guard's exit status becomes successful.
    Until,
}

/// A grouping of a list of commands and any comments trailing after the commands.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct CommandGroup<C> {
    /// The sequential list of commands.
    pub commands: Vec<C>,
    /// Any trailing comments appearing on the next line after the last command.
    pub trailing_comments: Vec<Newline>,
}

/// A grouping of guard and body commands, and any comments they may have.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct GuardBodyPairGroup<C> {
    /// The guard commands, which if successful, should lead to the
    /// execution of the body commands.
    pub guard: CommandGroup<C>,
    /// The body commands to execute if the guard is successful.
    pub body: CommandGroup<C>,
}

/// Parsed fragments relating to a shell `if` command.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct IfFragments<C> {
    /// A list of conditionals branches.
    pub conditionals: Vec<GuardBodyPairGroup<C>>,
    /// The `else` branch, if any,
    pub else_branch: Option<CommandGroup<C>>,
}

/// Parsed fragments relating to a shell `for` command.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct ForFragments<W, C> {
    /// The name of the variable to which each of the words will be bound.
    pub var: String,
    /// A comment that begins on the same line as the variable declaration.
    pub var_comment: Option<Newline>,
    /// Any comments after the variable declaration, a group of words to
    /// iterator over, and comment defined on the same line as the words.
    pub words: Option<(Vec<Newline>, Vec<W>, Option<Newline>)>,
    /// Any comments that appear after the `words` declaration (if it exists),
    /// but before the body of commands.
    pub pre_body_comments: Vec<Newline>,
    /// The body to be invoked for every iteration.
    pub body: CommandGroup<C>,
}

/// Parsed fragments relating to a shell `case` command.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct CaseFragments<W, C> {
    /// The word to be matched against.
    pub word: W,
    /// The comments appearing after the word to match but before the `in` reserved word.
    pub post_word_comments: Vec<Newline>,
    /// A comment appearing immediately after the `in` reserved word,
    /// yet still on the same line.
    pub in_comment: Option<Newline>,
    /// All the possible branches of the `case` command.
    pub arms: Vec<CaseArm<W, C>>,
    /// The comments appearing after the last arm but before the `esac` reserved word.
    pub post_arms_comments: Vec<Newline>,
}

/// An individual "unit of execution" within a `case` command.
///
/// Each arm has a number of pattern alternatives, and a body
/// of commands to run if any pattern matches.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct CaseArm<W, C> {
    /// The patterns which correspond to this case arm.
    pub patterns: CasePatternFragments<W>,
    /// The body of commands to run if any pattern matches.
    pub body: CommandGroup<C>,
    /// A comment appearing at the end of the arm declaration,
    /// i.e. after `;;` but on the same line.
    pub arm_comment: Option<Newline>,
}

/// Parsed fragments relating to patterns in a shell `case` command.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct CasePatternFragments<W> {
    /// Comments appearing after a previous arm, but before the start of a pattern.
    pub pre_pattern_comments: Vec<Newline>,
    /// Pattern alternatives which all correspond to the same case arm.
    pub pattern_alternatives: Vec<W>,
    /// A comment appearing at the end of the pattern declaration on the same line.
    pub pattern_comment: Option<Newline>,
}

/// An indicator to the builder what kind of complex word was parsed.
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum ComplexWordKind<C> {
    /// Several distinct words concatenated together.
    Concat(Vec<WordKind<C>>),
    /// A regular word.
    Single(WordKind<C>),
}

/// An indicator to the builder what kind of word was parsed.
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum WordKind<C> {
    /// A regular word.
    Simple(SimpleWordKind<C>),
    /// List of words concatenated within double quotes.
    DoubleQuoted(Vec<SimpleWordKind<C>>),
    /// List of words concatenated within single quotes. Virtually
    /// identical as a literal, but makes a distinction between the two.
    SingleQuoted(String),
}

/// An indicator to the builder what kind of simple word was parsed.
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum SimpleWordKind<C> {
    /// A non-special literal word.
    Literal(String),
    /// Access of a value inside a parameter, e.g. `$foo` or `$$`.
    Param(DefaultParameter),
    /// A parameter substitution, e.g. `${param-word}`.
    Subst(Box<ParameterSubstitutionKind<ComplexWordKind<C>, C>>),
    /// Represents the standard output of some command, e.g. \`echo foo\`.
    CommandSubst(CommandGroup<C>),
    /// A token which normally has a special meaning is treated as a literal
    /// because it was escaped, typically with a backslash, e.g. `\"`.
    Escaped(String),
    /// Represents `*`, useful for handling pattern expansions.
    Star,
    /// Represents `?`, useful for handling pattern expansions.
    Question,
    /// Represents `[`, useful for handling pattern expansions.
    SquareOpen,
    /// Represents `]`, useful for handling pattern expansions.
    SquareClose,
    /// Represents `~`, useful for handling tilde expansions.
    Tilde,
    /// Represents `:`, useful for handling tilde expansions.
    Colon,
}

/// Represents redirecting a command's file descriptors.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum RedirectKind<W> {
    /// Open a file for reading, e.g. `[n]< file`.
    Read(Option<u16>, W),
    /// Open a file for writing after truncating, e.g. `[n]> file`.
    Write(Option<u16>, W),
    /// Open a file for reading and writing, e.g. `[n]<> file`.
    ReadWrite(Option<u16>, W),
    /// Open a file for writing, appending to the end, e.g. `[n]>> file`.
    Append(Option<u16>, W),
    /// Open a file for writing, failing if the `noclobber` shell option is set, e.g. `[n]>| file`.
    Clobber(Option<u16>, W),
    /// Lines contained in the source that should be provided by as input to a file descriptor.
    Heredoc(Option<u16>, W),
    /// Duplicate a file descriptor for reading, e.g. `[n]<& [n|-]`.
    DupRead(Option<u16>, W),
    /// Duplicate a file descriptor for writing, e.g. `[n]>& [n|-]`.
    DupWrite(Option<u16>, W),
}

/// Represents the type of parameter that was parsed
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum ParameterSubstitutionKind<W, C> {
    /// Returns the standard output of running a command, e.g. `$(cmd)`
    Command(CommandGroup<C>),
    /// Returns the length of the value of a parameter, e.g. ${#param}
    Len(DefaultParameter),
    /// Returns the resulting value of an arithmetic subsitution, e.g. `$(( x++ ))`
    Arith(Option<DefaultArithmetic>),
    /// Use a provided value if the parameter is null or unset, e.g.
    /// `${param:-[word]}`.
    /// The boolean indicates the presence of a `:`, and that if the parameter has
    /// a null value, that situation should be treated as if the parameter is unset.
    Default(bool, DefaultParameter, Option<W>),
    /// Assign a provided value to the parameter if it is null or unset,
    /// e.g. `${param:=[word]}`.
    /// The boolean indicates the presence of a `:`, and that if the parameter has
    /// a null value, that situation should be treated as if the parameter is unset.
    Assign(bool, DefaultParameter, Option<W>),
    /// If the parameter is null or unset, an error should result with the provided
    /// message, e.g. `${param:?[word]}`.
    /// The boolean indicates the presence of a `:`, and that if the parameter has
    /// a null value, that situation should be treated as if the parameter is unset.
    Error(bool, DefaultParameter, Option<W>),
    /// If the parameter is NOT null or unset, a provided word will be used,
    /// e.g. `${param:+[word]}`.
    /// The boolean indicates the presence of a `:`, and that if the parameter has
    /// a null value, that situation should be treated as if the parameter is unset.
    Alternative(bool, DefaultParameter, Option<W>),
    /// Remove smallest suffix pattern, e.g. `${param%pattern}`
    RemoveSmallestSuffix(DefaultParameter, Option<W>),
    /// Remove largest suffix pattern, e.g. `${param%%pattern}`
    RemoveLargestSuffix(DefaultParameter, Option<W>),
    /// Remove smallest prefix pattern, e.g. `${param#pattern}`
    RemoveSmallestPrefix(DefaultParameter, Option<W>),
    /// Remove largest prefix pattern, e.g. `${param##pattern}`
    RemoveLargestPrefix(DefaultParameter, Option<W>),
}

/// Represents a parsed newline, more specifically, the presense of a comment
/// immediately preceeding the newline.
///
/// Since shell comments are usually treated as a newline, they can be present
/// anywhere a newline can be as well. Thus if it is desired to retain comments
/// they can be optionally attached to a parsed newline.
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Newline(pub Option<String>);

/// A trait which defines an interface which the parser defined in the `parse` module
/// uses to delegate Abstract Syntax Tree creation. The methods defined here correspond
/// to their respectively named methods on the parser, and accept the relevant data for
/// each shell command type.
pub trait Builder {
    /// The type which represents a complete, top-level command.
    type Command;
    /// The type which represents an and/or list of commands.
    type CommandList;
    /// The type which represents a command that can be used in an and/or command list.
    type ListableCommand;
    /// The type which represents a command that can be used in a pipeline.
    type PipeableCommand;
    /// The type which represents compound commands like `if`, `case`, `for`, etc.
    type CompoundCommand;
    /// The type which represents shell words, which can be command names or arguments.
    type Word;
    /// The type which represents a file descriptor redirection.
    type Redirect;
    /// A type for returning custom parse/build errors.
    type Error;

    /// Invoked once a complete command is found. That is, a command delimited by a
    /// newline, semicolon, ampersand, or the end of input.
    ///
    /// # Arguments
    /// * pre_cmd_comments: any comments that appear before the start of the command
    /// * list: an and/or list of commands previously generated by the same builder
    /// * separator: indicates how the command was delimited
    /// * cmd_comment: a comment that appears at the end of the command
    fn complete_command(
        &mut self,
        pre_cmd_comments: Vec<Newline>,
        list: Self::CommandList,
        separator: SeparatorKind,
        cmd_comment: Option<Newline>,
    ) -> Result<Self::Command, Self::Error>;

    /// Invoked when multiple commands are parsed which are separated by `&&` or `||`.
    /// Typically after the first command is run, each of the following commands may or
    /// may not be executed, depending on the exit status of the previously executed command.
    ///
    /// # Arguments
    /// * first: the first command before any `&&` or `||` separator
    /// * rest: A collection of comments after the last separator and the next command.
    fn and_or_list(
        &mut self,
        first: Self::ListableCommand,
        rest: Vec<(Vec<Newline>, AndOr<Self::ListableCommand>)>,
    ) -> Result<Self::CommandList, Self::Error>;

    /// Invoked when a pipeline of commands is parsed.
    /// A pipeline is one or more commands where the standard output of the previous
    /// typically becomes the standard input of the next.
    ///
    /// # Arguments
    /// * bang: the presence of a `!` at the start of the pipeline, typically indicating
    /// that the pipeline's exit status should be logically inverted.
    /// * cmds: a collection of tuples which are any comments appearing after a pipe token, followed
    /// by the command itself, all in the order they were parsed
    fn pipeline(
        &mut self,
        bang: bool,
        cmds: Vec<(Vec<Newline>, Self::PipeableCommand)>,
    ) -> Result<Self::ListableCommand, Self::Error>;

    /// Invoked when the "simplest" possible command is parsed: an executable with arguments.
    ///
    /// # Arguments
    /// * redirects_or_env_vars: redirections or environment variables that occur before any command
    /// * redirects_or_cmd_words: redirections or any command or argument
    fn simple_command(
        &mut self,
        redirects_or_env_vars: Vec<RedirectOrEnvVar<Self::Redirect, String, Self::Word>>,
        redirects_or_cmd_words: Vec<RedirectOrCmdWord<Self::Redirect, Self::Word>>,
    ) -> Result<Self::PipeableCommand, Self::Error>;

    /// Invoked when a non-zero number of commands were parsed between balanced curly braces.
    /// Typically these commands should run within the current shell environment.
    ///
    /// # Arguments
    /// * cmds: the commands that were parsed between braces
    /// * redirects: any redirects to be applied over the **entire** group of commands
    fn brace_group(
        &mut self,
        cmds: CommandGroup<Self::Command>,
        redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error>;

    /// Invoked when a non-zero number of commands were parsed between balanced parentheses.
    /// Typically these commands should run within their own environment without affecting
    /// the shell's global environment.
    ///
    /// # Arguments
    /// * cmds: the commands that were parsed between parens
    /// * redirects: any redirects to be applied over the **entire** group of commands
    fn subshell(
        &mut self,
        cmds: CommandGroup<Self::Command>,
        redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error>;

    /// Invoked when a loop command like `while` or `until` is parsed.
    /// Typically these commands will execute their body based on the exit status of their guard.
    ///
    /// # Arguments
    /// * kind: the type of the loop: `while` or `until`
    /// * guard: commands that determine how long the loop will run for
    /// * body: commands to be run every iteration of the loop
    /// * redirects: any redirects to be applied over **all** commands part of the loop
    fn loop_command(
        &mut self,
        kind: LoopKind,
        guard_body_pair: GuardBodyPairGroup<Self::Command>,
        redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error>;

    /// Invoked when an `if` conditional command is parsed.
    /// Typically an `if` command is made up of one or more guard-body pairs, where the body
    /// of the first successful corresponding guard is executed. There can also be an optional
    /// `else` part to be run if no guard is successful.
    ///
    /// # Arguments
    /// * fragments: parsed fragments relating to a shell `if` command.
    /// * redirects: any redirects to be applied over **all** commands within the `if` command
    fn if_command(
        &mut self,
        fragments: IfFragments<Self::Command>,
        redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error>;

    /// Invoked when a `for` command is parsed.
    /// Typically a `for` command binds a variable to each member in a group of words and
    /// invokes its body with that variable present in the environment. If no words are
    /// specified, the command will iterate over the arguments to the script or enclosing function.
    ///
    /// # Arguments
    /// * fragments: parsed fragments relating to a shell `for` command.
    /// * redirects: any redirects to be applied over **all** commands within the `for` command
    fn for_command(
        &mut self,
        fragments: ForFragments<Self::Word, Self::Command>,
        redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error>;

    /// Invoked when a `case` command is parsed.
    /// Typically this command will execute certain commands when a given word matches a pattern.
    ///
    /// # Arguments
    /// * fragments: parsed fragments relating to a shell `case` command.
    /// * redirects: any redirects to be applied over **all** commands part of the `case` block
    fn case_command(
        &mut self,
        fragments: CaseFragments<Self::Word, Self::Command>,
        redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error>;

    /// Bridges the gap between a `PipeableCommand` and a `CompoundCommand` since
    /// `CompoundCommand`s are typically `PipeableCommand`s as well.
    ///
    /// # Arguments
    /// cmd: The `CompoundCommand` to convert into a `PipeableCommand`
    fn compound_command_into_pipeable(
        &mut self,
        cmd: Self::CompoundCommand,
    ) -> Result<Self::PipeableCommand, Self::Error>;

    /// Invoked when a function declaration is parsed.
    /// Typically a function declaration overwrites any previously defined function
    /// within the current environment.
    ///
    /// # Arguments
    /// * name: the name of the function to be created
    /// * post_name_comments: any comments appearing after the function name but before the body
    /// * body: commands to be run when the function is invoked
    fn function_declaration(
        &mut self,
        name: String,
        post_name_comments: Vec<Newline>,
        body: Self::CompoundCommand,
    ) -> Result<Self::PipeableCommand, Self::Error>;

    /// Invoked when only comments are parsed with no commands following.
    /// This can occur if an entire shell script is commented out or if there
    /// are comments present at the end of the script.
    ///
    /// # Arguments
    /// * comments: the parsed comments
    fn comments(&mut self, comments: Vec<Newline>) -> Result<(), Self::Error>;

    /// Invoked when a word is parsed.
    ///
    /// # Arguments
    /// * kind: the type of word that was parsed
    fn word(&mut self, kind: ComplexWordKind<Self::Command>) -> Result<Self::Word, Self::Error>;

    /// Invoked when a redirect is parsed.
    ///
    /// # Arguments
    /// * kind: the type of redirect that was parsed
    fn redirect(&mut self, kind: RedirectKind<Self::Word>) -> Result<Self::Redirect, Self::Error>;
}

macro_rules! impl_builder_body {
    ($T:ident) => {
        type Command = $T::Command;
        type CommandList = $T::CommandList;
        type ListableCommand = $T::ListableCommand;
        type PipeableCommand = $T::PipeableCommand;
        type CompoundCommand = $T::CompoundCommand;
        type Word = $T::Word;
        type Redirect = $T::Redirect;
        type Error = $T::Error;

        fn complete_command(
            &mut self,
            pre_cmd_comments: Vec<Newline>,
            list: Self::CommandList,
            separator: SeparatorKind,
            cmd_comment: Option<Newline>,
        ) -> Result<Self::Command, Self::Error> {
            (**self).complete_command(pre_cmd_comments, list, separator, cmd_comment)
        }

        fn and_or_list(
            &mut self,
            first: Self::ListableCommand,
            rest: Vec<(Vec<Newline>, AndOr<Self::ListableCommand>)>,
        ) -> Result<Self::CommandList, Self::Error> {
            (**self).and_or_list(first, rest)
        }

        fn pipeline(
            &mut self,
            bang: bool,
            cmds: Vec<(Vec<Newline>, Self::PipeableCommand)>,
        ) -> Result<Self::ListableCommand, Self::Error> {
            (**self).pipeline(bang, cmds)
        }

        fn simple_command(
            &mut self,
            redirects_or_env_vars: Vec<RedirectOrEnvVar<Self::Redirect, String, Self::Word>>,
            redirects_or_cmd_words: Vec<RedirectOrCmdWord<Self::Redirect, Self::Word>>,
        ) -> Result<Self::PipeableCommand, Self::Error> {
            (**self).simple_command(redirects_or_env_vars, redirects_or_cmd_words)
        }

        fn brace_group(
            &mut self,
            cmds: CommandGroup<Self::Command>,
            redirects: Vec<Self::Redirect>,
        ) -> Result<Self::CompoundCommand, Self::Error> {
            (**self).brace_group(cmds, redirects)
        }

        fn subshell(
            &mut self,
            cmds: CommandGroup<Self::Command>,
            redirects: Vec<Self::Redirect>,
        ) -> Result<Self::CompoundCommand, Self::Error> {
            (**self).subshell(cmds, redirects)
        }

        fn loop_command(
            &mut self,
            kind: LoopKind,
            guard_body_pair: GuardBodyPairGroup<Self::Command>,
            redirects: Vec<Self::Redirect>,
        ) -> Result<Self::CompoundCommand, Self::Error> {
            (**self).loop_command(kind, guard_body_pair, redirects)
        }

        fn if_command(
            &mut self,
            fragments: IfFragments<Self::Command>,
            redirects: Vec<Self::Redirect>,
        ) -> Result<Self::CompoundCommand, Self::Error> {
            (**self).if_command(fragments, redirects)
        }

        fn for_command(
            &mut self,
            fragments: ForFragments<Self::Word, Self::Command>,
            redirects: Vec<Self::Redirect>,
        ) -> Result<Self::CompoundCommand, Self::Error> {
            (**self).for_command(fragments, redirects)
        }

        fn case_command(
            &mut self,
            fragments: CaseFragments<Self::Word, Self::Command>,
            redirects: Vec<Self::Redirect>,
        ) -> Result<Self::CompoundCommand, Self::Error> {
            (**self).case_command(fragments, redirects)
        }

        fn compound_command_into_pipeable(
            &mut self,
            cmd: Self::CompoundCommand,
        ) -> Result<Self::PipeableCommand, Self::Error> {
            (**self).compound_command_into_pipeable(cmd)
        }

        fn function_declaration(
            &mut self,
            name: String,
            post_name_comments: Vec<Newline>,
            body: Self::CompoundCommand,
        ) -> Result<Self::PipeableCommand, Self::Error> {
            (**self).function_declaration(name, post_name_comments, body)
        }

        fn comments(&mut self, comments: Vec<Newline>) -> Result<(), Self::Error> {
            (**self).comments(comments)
        }

        fn word(
            &mut self,
            kind: ComplexWordKind<Self::Command>,
        ) -> Result<Self::Word, Self::Error> {
            (**self).word(kind)
        }

        fn redirect(
            &mut self,
            kind: RedirectKind<Self::Word>,
        ) -> Result<Self::Redirect, Self::Error> {
            (**self).redirect(kind)
        }
    };
}

impl<'a, T: Builder + ?Sized> Builder for &'a mut T {
    impl_builder_body!(T);
}

impl<T: Builder + ?Sized> Builder for Box<T> {
    impl_builder_body!(T);
}


use crate::ast::builder::*;
use crate::ast::*;
use std::default::Default;
use std::fmt;
use std::marker::PhantomData;
use std::rc::Rc;
use std::sync::Arc;
use void::Void;

/// A macro for defining a default builder, its boilerplate, and delegating
/// the `Builder` trait to its respective `CoreBuilder` type.
///
/// This allows us to create concrete atomic/non-atomic builders which
/// wrap a concrete `CoreBuilder` implementation so we can hide its type
/// complexity from the consumer.
///
/// We could accomplish this by using a public type alias to the private
/// builder type, however, rustdoc will only generate docs for the alias
/// definition is, and the docs for the inner builder will be rendered in
/// their entire complexity.
// FIXME: might be good to revisit this complexity/indirection
macro_rules! default_builder {
    ($(#[$attr:meta])*
     pub struct $Builder:ident,
     $CoreBuilder:ident,
     $Word:ident,
     $Cmd:ident,
     $PipeableCmd:ident,
    ) => {
        $(#[$attr])*
        pub struct $Builder<T>($CoreBuilder<T, $Word<T>, $Cmd<T>>);

        impl<T> $Builder<T> {
            /// Constructs a builder.
            pub fn new() -> Self {
                $Builder($CoreBuilder::new())
            }
        }

        impl<T> fmt::Debug for $Builder<T> {
            fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
                fmt.debug_struct(stringify!($Builder))
                    .finish()
            }
        }

        impl<T> Default for $Builder<T> {
            fn default() -> Self {
                Self::new()
            }
        }

        impl<T> Clone for $Builder<T> {
            fn clone(&self) -> Self {
                *self
            }
        }

        impl<T> Copy for $Builder<T> {}

        impl<T: From<String>> Builder for $Builder<T> {
            type Command         = $Cmd<T>;
            type CommandList     = AndOrList<Self::ListableCommand>;
            type ListableCommand = ListableCommand<Self::PipeableCommand>;
            type PipeableCommand = $PipeableCmd<T, Self::Word, Self::Command>;
            type CompoundCommand = ShellCompoundCommand<T, Self::Word, Self::Command>;
            type Word            = $Word<T>;
            type Redirect        = Redirect<Self::Word>;
            type Error           = Void;

            fn complete_command(&mut self,
                                pre_cmd_comments: Vec<Newline>,
                                list: Self::CommandList,
                                separator: SeparatorKind,
                                cmd_comment: Option<Newline>)
                -> Result<Self::Command, Self::Error>
            {
                self.0.complete_command(pre_cmd_comments, list, separator, cmd_comment)
            }

            fn and_or_list(&mut self,
                      first: Self::ListableCommand,
                      rest: Vec<(Vec<Newline>, AndOr<Self::ListableCommand>)>)
                -> Result<Self::CommandList, Self::Error>
            {
                self.0.and_or_list(first, rest)
            }

            fn pipeline(&mut self,
                        bang: bool,
                        cmds: Vec<(Vec<Newline>, Self::PipeableCommand)>)
                -> Result<Self::ListableCommand, Self::Error>
            {
                self.0.pipeline(bang, cmds)
            }

            fn simple_command(
                &mut self,
                redirects_or_env_vars: Vec<RedirectOrEnvVar<Self::Redirect, String, Self::Word>>,
                redirects_or_cmd_words: Vec<RedirectOrCmdWord<Self::Redirect, Self::Word>>
            ) -> Result<Self::PipeableCommand, Self::Error>
            {
                self.0.simple_command(redirects_or_env_vars, redirects_or_cmd_words)
            }

            fn brace_group(&mut self,
                           cmds: CommandGroup<Self::Command>,
                           redirects: Vec<Self::Redirect>)
                -> Result<Self::CompoundCommand, Self::Error>
            {
                self.0.brace_group(cmds, redirects)
            }

            fn subshell(&mut self,
                        cmds: CommandGroup<Self::Command>,
                        redirects: Vec<Self::Redirect>)
                -> Result<Self::CompoundCommand, Self::Error>
            {
                self.0.subshell(cmds, redirects)
            }

            fn loop_command(&mut self,
                            kind: LoopKind,
                            guard_body_pair: GuardBodyPairGroup<Self::Command>,
                            redirects: Vec<Self::Redirect>)
                -> Result<Self::CompoundCommand, Self::Error>
            {
                self.0.loop_command(kind, guard_body_pair, redirects)
            }

            fn if_command(&mut self,
                          fragments: IfFragments<Self::Command>,
                          redirects: Vec<Self::Redirect>)
                -> Result<Self::CompoundCommand, Self::Error>
            {
                self.0.if_command(fragments, redirects)
            }

            fn for_command(&mut self,
                           fragments: ForFragments<Self::Word, Self::Command>,
                           redirects: Vec<Self::Redirect>)
                -> Result<Self::CompoundCommand, Self::Error>
            {
                self.0.for_command(fragments, redirects)
            }

            fn case_command(&mut self,
                            fragments: CaseFragments<Self::Word, Self::Command>,
                            redirects: Vec<Self::Redirect>)
                -> Result<Self::CompoundCommand, Self::Error>
            {
                self.0.case_command(fragments, redirects)
            }

            fn compound_command_into_pipeable(&mut self,
                                              cmd: Self::CompoundCommand)
                -> Result<Self::PipeableCommand, Self::Error>
            {
                self.0.compound_command_into_pipeable(cmd)
            }

            fn function_declaration(&mut self,
                                    name: String,
                                    post_name_comments: Vec<Newline>,
                                    body: Self::CompoundCommand)
                -> Result<Self::PipeableCommand, Self::Error>
            {
                self.0.function_declaration(name, post_name_comments, body)
            }

            fn comments(&mut self,
                        comments: Vec<Newline>)
                -> Result<(), Self::Error>
            {
                self.0.comments(comments)
            }

            fn word(&mut self,
                    kind: ComplexWordKind<Self::Command>)
                -> Result<Self::Word, Self::Error>
            {
                self.0.word(kind)
            }

            fn redirect(&mut self,
                        kind: RedirectKind<Self::Word>)
                -> Result<Self::Redirect, Self::Error>
            {
                self.0.redirect(kind)
            }
        }
    };
}

type RcCoreBuilder<T, W, C> = CoreBuilder<T, W, C, Rc<ShellCompoundCommand<T, W, C>>>;
type ArcCoreBuilder<T, W, C> = CoreBuilder<T, W, C, Arc<ShellCompoundCommand<T, W, C>>>;

default_builder! {
    /// A `Builder` implementation which builds shell commands
    /// using the (non-atomic) AST definitions in the `ast` module.
    pub struct DefaultBuilder,
    RcCoreBuilder,
    TopLevelWord,
    TopLevelCommand,
    ShellPipeableCommand,
}

default_builder! {
    /// A `Builder` implementation which builds shell commands
    /// using the (atomic) AST definitions in the `ast` module.
    pub struct AtomicDefaultBuilder,
    ArcCoreBuilder,
    AtomicTopLevelWord,
    AtomicTopLevelCommand,
    AtomicShellPipeableCommand,
}

/// A `DefaultBuilder` implementation which uses regular `String`s when
/// representing shell words.
pub type StringBuilder = DefaultBuilder<String>;

/// A `DefaultBuilder` implementation which uses `Rc<String>`s when
/// representing shell words.
pub type RcBuilder = DefaultBuilder<Rc<String>>;

/// A `DefaultBuilder` implementation which uses `Arc<String>`s when
/// representing shell words.
pub type ArcBuilder = AtomicDefaultBuilder<Arc<String>>;

/// The actual provided `Builder` implementation.
/// The various type parameters are used to swap out atomic/non-atomic AST versions.
pub struct CoreBuilder<T, W, C, F> {
    phantom_data: PhantomData<(T, W, C, F)>,
}

impl<T, W, C, F> fmt::Debug for CoreBuilder<T, W, C, F> {
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        fmt.debug_struct("CoreBuilder").finish()
    }
}

impl<T, W, C, F> Clone for CoreBuilder<T, W, C, F> {
    fn clone(&self) -> Self {
        *self
    }
}

impl<T, W, C, F> Copy for CoreBuilder<T, W, C, F> {}

impl<T, W, C, F> Default for CoreBuilder<T, W, C, F> {
    fn default() -> Self {
        Self::new()
    }
}

impl<T, W, C, F> CoreBuilder<T, W, C, F> {
    /// Constructs a builder.
    pub fn new() -> Self {
        CoreBuilder {
            phantom_data: PhantomData,
        }
    }
}

type BuilderPipeableCommand<T, W, C, F> = PipeableCommand<
    T,
    Box<SimpleCommand<T, W, Redirect<W>>>,
    Box<ShellCompoundCommand<T, W, C>>,
    F,
>;

impl<T, W, C, F> Builder for CoreBuilder<T, W, C, F>
where
    T: From<String>,
    W: From<ShellWord<T, W, C>>,
    C: From<Command<AndOrList<ListableCommand<BuilderPipeableCommand<T, W, C, F>>>>>,
    F: From<ShellCompoundCommand<T, W, C>>,
{
    type Command = C;
    type CommandList = AndOrList<Self::ListableCommand>;
    type ListableCommand = ListableCommand<Self::PipeableCommand>;
    type PipeableCommand = BuilderPipeableCommand<T, W, C, F>;
    type CompoundCommand = ShellCompoundCommand<T, Self::Word, Self::Command>;
    type Word = W;
    type Redirect = Redirect<Self::Word>;
    type Error = Void;

    /// Constructs a `Command::Job` node with the provided inputs if the command
    /// was delimited by an ampersand or the command itself otherwise.
    fn complete_command(
        &mut self,
        _pre_cmd_comments: Vec<Newline>,
        list: Self::CommandList,
        separator: SeparatorKind,
        _cmd_comment: Option<Newline>,
    ) -> Result<Self::Command, Self::Error> {
        let cmd = match separator {
            SeparatorKind::Semi | SeparatorKind::Other | SeparatorKind::Newline => {
                Command::List(list)
            }
            SeparatorKind::Amp => Command::Job(list),
        };

        Ok(cmd.into())
    }

    /// Constructs a `Command::List` node with the provided inputs.
    fn and_or_list(
        &mut self,
        first: Self::ListableCommand,
        rest: Vec<(Vec<Newline>, AndOr<Self::ListableCommand>)>,
    ) -> Result<Self::CommandList, Self::Error> {
        Ok(AndOrList {
            first,
            rest: rest.into_iter().map(|(_, c)| c).collect(),
        })
    }

    /// Constructs a `Command::Pipe` node with the provided inputs or a `Command::Simple`
    /// node if only a single command with no status inversion is supplied.
    fn pipeline(
        &mut self,
        bang: bool,
        cmds: Vec<(Vec<Newline>, Self::PipeableCommand)>,
    ) -> Result<Self::ListableCommand, Self::Error> {
        debug_assert_eq!(cmds.is_empty(), false);
        let mut cmds: Vec<_> = cmds.into_iter().map(|(_, c)| c).collect();

        // Pipe is the only AST node which allows for a status
        // negation, so we are forced to use it even if we have a single
        // command. Otherwise there is no need to wrap it further.
        if bang || cmds.len() > 1 {
            cmds.shrink_to_fit();
            Ok(ListableCommand::Pipe(bang, cmds))
        } else {
            Ok(ListableCommand::Single(cmds.pop().unwrap()))
        }
    }

    /// Constructs a `Command::Simple` node with the provided inputs.
    fn simple_command(
        &mut self,
        redirects_or_env_vars: Vec<RedirectOrEnvVar<Self::Redirect, String, Self::Word>>,
        mut redirects_or_cmd_words: Vec<RedirectOrCmdWord<Self::Redirect, Self::Word>>,
    ) -> Result<Self::PipeableCommand, Self::Error> {
        let redirects_or_env_vars = redirects_or_env_vars
            .into_iter()
            .map(|roev| match roev {
                RedirectOrEnvVar::Redirect(red) => RedirectOrEnvVar::Redirect(red),
                RedirectOrEnvVar::EnvVar(k, v) => RedirectOrEnvVar::EnvVar(k.into(), v),
            })
            .collect();

        redirects_or_cmd_words.shrink_to_fit();

        Ok(PipeableCommand::Simple(Box::new(SimpleCommand {
            redirects_or_env_vars,
            redirects_or_cmd_words,
        })))
    }

    /// Constructs a `CompoundCommand::Brace` node with the provided inputs.
    fn brace_group(
        &mut self,
        cmd_group: CommandGroup<Self::Command>,
        mut redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        let mut cmds = cmd_group.commands;
        cmds.shrink_to_fit();
        redirects.shrink_to_fit();
        Ok(CompoundCommand {
            kind: CompoundCommandKind::Brace(cmds),
            io: redirects,
        })
    }

    /// Constructs a `CompoundCommand::Subshell` node with the provided inputs.
    fn subshell(
        &mut self,
        cmd_group: CommandGroup<Self::Command>,
        mut redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        let mut cmds = cmd_group.commands;
        cmds.shrink_to_fit();
        redirects.shrink_to_fit();
        Ok(CompoundCommand {
            kind: CompoundCommandKind::Subshell(cmds),
            io: redirects,
        })
    }

    /// Constructs a `CompoundCommand::Loop` node with the provided inputs.
    fn loop_command(
        &mut self,
        kind: LoopKind,
        guard_body_pair: GuardBodyPairGroup<Self::Command>,
        mut redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        let mut guard = guard_body_pair.guard.commands;
        let mut body = guard_body_pair.body.commands;

        guard.shrink_to_fit();
        body.shrink_to_fit();
        redirects.shrink_to_fit();

        let guard_body_pair = GuardBodyPair { guard, body };

        let loop_cmd = match kind {
            LoopKind::While => CompoundCommandKind::While(guard_body_pair),
            LoopKind::Until => CompoundCommandKind::Until(guard_body_pair),
        };

        Ok(CompoundCommand {
            kind: loop_cmd,
            io: redirects,
        })
    }

    /// Constructs a `CompoundCommand::If` node with the provided inputs.
    fn if_command(
        &mut self,
        fragments: IfFragments<Self::Command>,
        mut redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        let IfFragments {
            conditionals,
            else_branch,
        } = fragments;

        let conditionals = conditionals
            .into_iter()
            .map(|gbp| {
                let mut guard = gbp.guard.commands;
                let mut body = gbp.body.commands;

                guard.shrink_to_fit();
                body.shrink_to_fit();

                GuardBodyPair { guard, body }
            })
            .collect();

        let else_branch = else_branch.map(
            |CommandGroup {
                 commands: mut els, ..
             }| {
                els.shrink_to_fit();
                els
            },
        );

        redirects.shrink_to_fit();

        Ok(CompoundCommand {
            kind: CompoundCommandKind::If {
                conditionals,
                else_branch,
            },
            io: redirects,
        })
    }

    /// Constructs a `CompoundCommand::For` node with the provided inputs.
    fn for_command(
        &mut self,
        fragments: ForFragments<Self::Word, Self::Command>,
        mut redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        let words = fragments.words.map(|(_, mut words, _)| {
            words.shrink_to_fit();
            words
        });

        let mut body = fragments.body.commands;
        body.shrink_to_fit();
        redirects.shrink_to_fit();

        Ok(CompoundCommand {
            kind: CompoundCommandKind::For {
                var: fragments.var.into(),
                words,
                body,
            },
            io: redirects,
        })
    }

    /// Constructs a `CompoundCommand::Case` node with the provided inputs.
    fn case_command(
        &mut self,
        fragments: CaseFragments<Self::Word, Self::Command>,
        mut redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        let arms = fragments
            .arms
            .into_iter()
            .map(|arm| {
                let mut patterns = arm.patterns.pattern_alternatives;
                patterns.shrink_to_fit();

                let mut body = arm.body.commands;
                body.shrink_to_fit();

                PatternBodyPair { patterns, body }
            })
            .collect();

        redirects.shrink_to_fit();
        Ok(CompoundCommand {
            kind: CompoundCommandKind::Case {
                word: fragments.word,
                arms,
            },
            io: redirects,
        })
    }

    /// Converts a `CompoundCommand` into a `PipeableCommand`.
    fn compound_command_into_pipeable(
        &mut self,
        cmd: Self::CompoundCommand,
    ) -> Result<Self::PipeableCommand, Self::Error> {
        Ok(PipeableCommand::Compound(Box::new(cmd)))
    }

    /// Constructs a `Command::FunctionDef` node with the provided inputs.
    fn function_declaration(
        &mut self,
        name: String,
        _post_name_comments: Vec<Newline>,
        body: Self::CompoundCommand,
    ) -> Result<Self::PipeableCommand, Self::Error> {
        Ok(PipeableCommand::FunctionDef(name.into(), body.into()))
    }

    /// Ignored by the builder.
    fn comments(&mut self, _comments: Vec<Newline>) -> Result<(), Self::Error> {
        Ok(())
    }

    /// Constructs a `ast::Word` from the provided input.
    fn word(&mut self, kind: ComplexWordKind<Self::Command>) -> Result<Self::Word, Self::Error> {
        macro_rules! map {
            ($pat:expr) => {
                match $pat {
                    Some(w) => Some(self.word(w)?),
                    None => None,
                }
            };
        }

        fn map_arith<T: From<String>>(kind: DefaultArithmetic) -> Arithmetic<T> {
            use crate::ast::Arithmetic::*;
            match kind {
                Var(v) => Var(v.into()),
                Literal(l) => Literal(l),
                Pow(a, b) => Pow(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                PostIncr(p) => PostIncr(p.into()),
                PostDecr(p) => PostDecr(p.into()),
                PreIncr(p) => PreIncr(p.into()),
                PreDecr(p) => PreDecr(p.into()),
                UnaryPlus(a) => UnaryPlus(Box::new(map_arith(*a))),
                UnaryMinus(a) => UnaryMinus(Box::new(map_arith(*a))),
                LogicalNot(a) => LogicalNot(Box::new(map_arith(*a))),
                BitwiseNot(a) => BitwiseNot(Box::new(map_arith(*a))),
                Mult(a, b) => Mult(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Div(a, b) => Div(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Modulo(a, b) => Modulo(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Add(a, b) => Add(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Sub(a, b) => Sub(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                ShiftLeft(a, b) => ShiftLeft(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                ShiftRight(a, b) => ShiftRight(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Less(a, b) => Less(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                LessEq(a, b) => LessEq(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Great(a, b) => Great(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                GreatEq(a, b) => GreatEq(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Eq(a, b) => Eq(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                NotEq(a, b) => NotEq(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                BitwiseAnd(a, b) => BitwiseAnd(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                BitwiseXor(a, b) => BitwiseXor(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                BitwiseOr(a, b) => BitwiseOr(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                LogicalAnd(a, b) => LogicalAnd(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                LogicalOr(a, b) => LogicalOr(Box::new(map_arith(*a)), Box::new(map_arith(*b))),
                Ternary(a, b, c) => Ternary(
                    Box::new(map_arith(*a)),
                    Box::new(map_arith(*b)),
                    Box::new(map_arith(*c)),
                ),
                Assign(v, a) => Assign(v.into(), Box::new(map_arith(*a))),
                Sequence(ariths) => Sequence(ariths.into_iter().map(map_arith).collect()),
            }
        }

        let map_param = |kind: DefaultParameter| -> Parameter<T> {
            use crate::ast::Parameter::*;
            match kind {
                At => At,
                Star => Star,
                Pound => Pound,
                Question => Question,
                Dash => Dash,
                Dollar => Dollar,
                Bang => Bang,
                Positional(p) => Positional(p),
                Var(v) => Var(v.into()),
            }
        };

        let mut map_simple = |kind| {
            use crate::ast::builder::ParameterSubstitutionKind::*;

            let simple = match kind {
                SimpleWordKind::Literal(s) => SimpleWord::Literal(s.into()),
                SimpleWordKind::Escaped(s) => SimpleWord::Escaped(s.into()),
                SimpleWordKind::Param(p) => SimpleWord::Param(map_param(p)),
                SimpleWordKind::Star => SimpleWord::Star,
                SimpleWordKind::Question => SimpleWord::Question,
                SimpleWordKind::SquareOpen => SimpleWord::SquareOpen,
                SimpleWordKind::SquareClose => SimpleWord::SquareClose,
                SimpleWordKind::Tilde => SimpleWord::Tilde,
                SimpleWordKind::Colon => SimpleWord::Colon,

                SimpleWordKind::CommandSubst(c) => {
                    SimpleWord::Subst(Box::new(ParameterSubstitution::Command(c.commands)))
                }

                SimpleWordKind::Subst(s) => {
                    // Force a move out of the boxed substitution. For some reason doing
                    // the deref in the match statment gives a strange borrow failure
                    let s = *s;
                    let subst = match s {
                        Len(p) => ParameterSubstitution::Len(map_param(p)),
                        Command(c) => ParameterSubstitution::Command(c.commands),
                        Arith(a) => ParameterSubstitution::Arith(a.map(map_arith)),
                        Default(c, p, w) => {
                            ParameterSubstitution::Default(c, map_param(p), map!(w))
                        }
                        Assign(c, p, w) => ParameterSubstitution::Assign(c, map_param(p), map!(w)),
                        Error(c, p, w) => ParameterSubstitution::Error(c, map_param(p), map!(w)),
                        Alternative(c, p, w) => {
                            ParameterSubstitution::Alternative(c, map_param(p), map!(w))
                        }
                        RemoveSmallestSuffix(p, w) => {
                            ParameterSubstitution::RemoveSmallestSuffix(map_param(p), map!(w))
                        }
                        RemoveLargestSuffix(p, w) => {
                            ParameterSubstitution::RemoveLargestSuffix(map_param(p), map!(w))
                        }
                        RemoveSmallestPrefix(p, w) => {
                            ParameterSubstitution::RemoveSmallestPrefix(map_param(p), map!(w))
                        }
                        RemoveLargestPrefix(p, w) => {
                            ParameterSubstitution::RemoveLargestPrefix(map_param(p), map!(w))
                        }
                    };
                    SimpleWord::Subst(Box::new(subst))
                }
            };
            Ok(simple)
        };

        let mut map_word = |kind| {
            let word = match kind {
                WordKind::Simple(s) => Word::Simple(map_simple(s)?),
                WordKind::SingleQuoted(s) => Word::SingleQuoted(s.into()),
                WordKind::DoubleQuoted(v) => Word::DoubleQuoted(
                    v.into_iter()
                        .map(&mut map_simple)
                        .collect::<Result<Vec<_>, _>>()?,
                ),
            };
            Ok(word)
        };

        let word = match compress(kind) {
            ComplexWordKind::Single(s) => ComplexWord::Single(map_word(s)?),
            ComplexWordKind::Concat(words) => ComplexWord::Concat(
                words
                    .into_iter()
                    .map(map_word)
                    .collect::<Result<Vec<_>, _>>()?,
            ),
        };

        Ok(word.into())
    }

    /// Constructs a `ast::Redirect` from the provided input.
    fn redirect(&mut self, kind: RedirectKind<Self::Word>) -> Result<Self::Redirect, Self::Error> {
        let io = match kind {
            RedirectKind::Read(fd, path) => Redirect::Read(fd, path),
            RedirectKind::Write(fd, path) => Redirect::Write(fd, path),
            RedirectKind::ReadWrite(fd, path) => Redirect::ReadWrite(fd, path),
            RedirectKind::Append(fd, path) => Redirect::Append(fd, path),
            RedirectKind::Clobber(fd, path) => Redirect::Clobber(fd, path),
            RedirectKind::Heredoc(fd, body) => Redirect::Heredoc(fd, body),
            RedirectKind::DupRead(src, dst) => Redirect::DupRead(src, dst),
            RedirectKind::DupWrite(src, dst) => Redirect::DupWrite(src, dst),
        };

        Ok(io)
    }
}

#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
struct Coalesce<I: Iterator, F> {
    iter: I,
    cur: Option<I::Item>,
    func: F,
}

impl<I: Iterator, F> Coalesce<I, F> {
    fn new<T>(iter: T, func: F) -> Self
    where
        T: IntoIterator<IntoIter = I, Item = I::Item>,
    {
        Coalesce {
            iter: iter.into_iter(),
            cur: None,
            func,
        }
    }
}

type CoalesceResult<T> = Result<T, (T, T)>;
impl<I, F> Iterator for Coalesce<I, F>
where
    I: Iterator,
    F: FnMut(I::Item, I::Item) -> CoalesceResult<I::Item>,
{
    type Item = I::Item;

    fn next(&mut self) -> Option<Self::Item> {
        let cur = self.cur.take().or_else(|| self.iter.next());
        let (mut left, mut right) = match (cur, self.iter.next()) {
            (Some(l), Some(r)) => (l, r),
            (Some(l), None) | (None, Some(l)) => return Some(l),
            (None, None) => return None,
        };

        loop {
            match (self.func)(left, right) {
                Ok(combined) => match self.iter.next() {
                    Some(next) => {
                        left = combined;
                        right = next;
                    }
                    None => return Some(combined),
                },

                Err((left, right)) => {
                    debug_assert!(self.cur.is_none());
                    self.cur = Some(right);
                    return Some(left);
                }
            }
        }
    }
}

fn compress<C>(word: ComplexWordKind<C>) -> ComplexWordKind<C> {
    use crate::ast::builder::ComplexWordKind::*;
    use crate::ast::builder::SimpleWordKind::*;
    use crate::ast::builder::WordKind::*;

    fn coalesce_simple<C>(
        a: SimpleWordKind<C>,
        b: SimpleWordKind<C>,
    ) -> CoalesceResult<SimpleWordKind<C>> {
        match (a, b) {
            (Literal(mut a), Literal(b)) => {
                a.push_str(&b);
                Ok(Literal(a))
            }
            (a, b) => Err((a, b)),
        }
    }

    fn coalesce_word<C>(a: WordKind<C>, b: WordKind<C>) -> CoalesceResult<WordKind<C>> {
        match (a, b) {
            (Simple(a), Simple(b)) => coalesce_simple(a, b)
                .map(Simple)
                .map_err(|(a, b)| (Simple(a), Simple(b))),
            (SingleQuoted(mut a), SingleQuoted(b)) => {
                a.push_str(&b);
                Ok(SingleQuoted(a))
            }
            (DoubleQuoted(a), DoubleQuoted(b)) => {
                let quoted = Coalesce::new(a.into_iter().chain(b), coalesce_simple).collect();
                Ok(DoubleQuoted(quoted))
            }
            (a, b) => Err((a, b)),
        }
    }

    match word {
        Single(s) => Single(match s {
            s @ Simple(_) | s @ SingleQuoted(_) => s,
            DoubleQuoted(v) => DoubleQuoted(Coalesce::new(v, coalesce_simple).collect()),
        }),
        Concat(v) => {
            let mut body: Vec<_> = Coalesce::new(v.into_iter(), coalesce_word).collect();
            if body.len() == 1 {
                Single(body.pop().unwrap())
            } else {
                Concat(body)
            }
        }
    }
}


use crate::ast::builder::*;
use crate::ast::{AndOr, RedirectOrCmdWord, RedirectOrEnvVar};
use void::Void;

/// A no-op `Builder` which ignores all inputs and always returns `()`.
///
/// Useful for validation of correct programs (i.e. parsing input without
/// caring about the actual AST representations).
#[derive(Debug, Copy, Clone)]
pub struct EmptyBuilder;

impl Default for EmptyBuilder {
    fn default() -> Self {
        EmptyBuilder::new()
    }
}

impl EmptyBuilder {
    /// Constructs a builder.
    pub fn new() -> Self {
        EmptyBuilder
    }
}

impl Builder for EmptyBuilder {
    type Command = ();
    type CommandList = ();
    type ListableCommand = ();
    type PipeableCommand = ();
    type CompoundCommand = ();
    type Word = ();
    type Redirect = ();
    type Error = Void;

    fn complete_command(
        &mut self,
        _pre_cmd_comments: Vec<Newline>,
        _cmd: Self::Command,
        _separator: SeparatorKind,
        _cmd_comment: Option<Newline>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn and_or_list(
        &mut self,
        _first: Self::ListableCommand,
        _rest: Vec<(Vec<Newline>, AndOr<Self::ListableCommand>)>,
    ) -> Result<Self::CommandList, Self::Error> {
        Ok(())
    }

    fn pipeline(
        &mut self,
        _bang: bool,
        _cmds: Vec<(Vec<Newline>, Self::Command)>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn simple_command(
        &mut self,
        _redirects_or_env_vars: Vec<RedirectOrEnvVar<Self::Redirect, String, Self::Word>>,
        _redirects_or_cmd_words: Vec<RedirectOrCmdWord<Self::Redirect, Self::Word>>,
    ) -> Result<Self::PipeableCommand, Self::Error> {
        Ok(())
    }

    fn brace_group(
        &mut self,
        _cmds: CommandGroup<Self::Command>,
        _redirects: Vec<Self::Redirect>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn subshell(
        &mut self,
        _cmds: CommandGroup<Self::Command>,
        _redirects: Vec<Self::Redirect>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn loop_command(
        &mut self,
        __kind: LoopKind,
        __guard_body_pair: GuardBodyPairGroup<Self::Command>,
        __redirects: Vec<Self::Redirect>,
    ) -> Result<Self::CompoundCommand, Self::Error> {
        Ok(())
    }

    fn if_command(
        &mut self,
        _fragments: IfFragments<Self::Command>,
        _redirects: Vec<Self::Redirect>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn for_command(
        &mut self,
        _fragments: ForFragments<Self::Word, Self::Command>,
        _redirects: Vec<Self::Redirect>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn case_command(
        &mut self,
        _fragments: CaseFragments<Self::Word, Self::Command>,
        _redirects: Vec<Self::Redirect>,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn function_declaration(
        &mut self,
        _name: String,
        _post_name_comments: Vec<Newline>,
        _body: Self::CompoundCommand,
    ) -> Result<Self::Command, Self::Error> {
        Ok(())
    }

    fn comments(&mut self, _comments: Vec<Newline>) -> Result<(), Self::Error> {
        Ok(())
    }

    fn word(&mut self, _kind: ComplexWordKind<Self::Command>) -> Result<Self::Word, Self::Error> {
        Ok(())
    }

    fn redirect(&mut self, _kind: RedirectKind<Self::Word>) -> Result<Self::Redirect, Self::Error> {
        Ok(())
    }

    fn compound_command_into_pipeable(
        &mut self,
        _cmd: Self::CompoundCommand,
    ) -> Result<Self::PipeableCommand, Self::Error> {
        Ok(())
    }
}


//! An module for easily iterating over a `Token` stream.

use crate::parse::SourcePos;
use crate::token::Token;
use crate::token::Token::*;
use std::iter as std_iter;
use std::mem;

/// Indicates an error such that EOF was encountered while some unmatched
/// tokens were still pending. The error stores the unmatched token
/// and the position where it appears in the source.
#[derive(Debug)]
pub struct UnmatchedError(pub Token, pub SourcePos);

/// An internal variant that indicates if a token should be yielded
/// or the current position updated to some value.
#[derive(Debug)]
enum TokenOrPos {
    /// A consumed token which should be yielded.
    Tok(Token),
    /// The current position should be updated to the contained value.
    Pos(SourcePos),
}

impl TokenOrPos {
    /// Returns `true` if `self` is a `Tok` value.
    #[inline]
    fn is_tok(&self) -> bool {
        match *self {
            TokenOrPos::Tok(_) => true,
            TokenOrPos::Pos(_) => false,
        }
    }
}

/// An iterator that can track its internal position in the stream.
pub trait PositionIterator: Iterator {
    /// Get the current position of the iterator.
    fn pos(&self) -> SourcePos;
}

impl<'a, T: PositionIterator> PositionIterator for &'a mut T {
    fn pos(&self) -> SourcePos {
        (**self).pos()
    }
}

/// An iterator that supports peeking a single element in the stream.
///
/// Identical to `std::iter::Peekable` but in a trait form.
pub trait PeekableIterator: Iterator {
    /// Peek at the next item, identical to `std::iter::Peekable::peek`.
    fn peek(&mut self) -> Option<&Self::Item>;
}

impl<'a, T: PeekableIterator> PeekableIterator for &'a mut T {
    fn peek(&mut self) -> Option<&Self::Item> {
        (**self).peek()
    }
}

impl<I: Iterator> PeekableIterator for std_iter::Peekable<I> {
    fn peek(&mut self) -> Option<&Self::Item> {
        std_iter::Peekable::peek(self)
    }
}

/// A marker trait that unifies `PeekableIterator` and `PositionIterator`.
pub trait PeekablePositionIterator: PeekableIterator + PositionIterator {}
impl<T: PeekableIterator + PositionIterator> PeekablePositionIterator for T {}

/// A convenience trait for converting `Token` iterators into other sub-iterators.
pub trait TokenIterator: Sized + PeekablePositionIterator<Item = Token> {
    /// Returns an iterator that yields at least one token, but continues to yield
    /// tokens until all matching cases of single/double quotes, backticks,
    /// ${ }, $( ), or ( ) are found.
    fn balanced(&mut self) -> Balanced<&mut Self> {
        Balanced::new(self, None)
    }

    /// Returns an iterator that yields tokens up to when a (closing) single quote
    /// is reached (assuming that the caller has reached the opening quote and
    /// wishes to continue up to but not including the closing quote).
    fn single_quoted(&mut self, pos: SourcePos) -> Balanced<&mut Self> {
        Balanced::new(self, Some((SingleQuote, pos)))
    }

    /// Returns an iterator that yields tokens up to when a (closing) double quote
    /// is reached (assuming that the caller has reached the opening quote and
    /// wishes to continue up to but not including the closing quote).
    fn double_quoted(&mut self, pos: SourcePos) -> Balanced<&mut Self> {
        Balanced::new(self, Some((DoubleQuote, pos)))
    }

    /// Returns an iterator that yields tokens up to when a (closing) backtick
    /// is reached (assuming that the caller has reached the opening backtick and
    /// wishes to continue up to but not including the closing backtick).
    fn backticked(&mut self, pos: SourcePos) -> Balanced<&mut Self> {
        Balanced::new(self, Some((Backtick, pos)))
    }

    /// Returns an iterator that yields tokens up to when a (closing) backtick
    /// is reached (assuming that the caller has reached the opening backtick and
    /// wishes to continue up to but not including the closing backtick).
    /// Any backslashes followed by \, $, or ` are removed from the stream.
    fn backticked_remove_backslashes(
        &mut self,
        pos: SourcePos,
    ) -> BacktickBackslashRemover<&mut Self> {
        BacktickBackslashRemover::new(self.backticked(pos))
    }
}

/// Convenience trait for `Token` iterators which could be "rewound" so that
/// they can yield tokens that were already pulled out of their stream.
trait RewindableTokenIterator {
    /// Rewind the iterator with the provided tokens. Vector should contain
    /// the tokens in the order they should be yielded.
    fn rewind(&mut self, tokens: Vec<TokenOrPos>);

    /// Grab the next token (or internal position) that should be buffered
    /// by the caller.
    fn next_token_or_pos(&mut self) -> Option<TokenOrPos>;
}

/// A Token iterator that keeps track of how many lines have been read.
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Debug)]
pub struct TokenIter<I> {
    /// The underlying token iterator being wrapped. Iterator is fused to avoid
    /// inconsistent behavior when doing multiple peek ahead operations.
    iter: std_iter::Fuse<I>,
    /// Any tokens that were previously yielded but to be consumed later, stored
    /// as a stack. Intersperced between the tokens are any changes to the current
    /// position that should be applied. This is useful for situations where the
    /// parser may have removed certain tokens (e.g. \ when unescaping), but we still
    /// want to keep track of token positions in the actual source.
    prev_buffered: Vec<TokenOrPos>,
    /// The current position in the source that we have consumed up to
    pos: SourcePos,
}

impl<I: Iterator<Item = Token>> PositionIterator for TokenIter<I> {
    fn pos(&self) -> SourcePos {
        self.pos
    }
}

impl<I: Iterator<Item = Token>> PeekableIterator for TokenIter<I> {
    fn peek(&mut self) -> Option<&Self::Item> {
        // Peek the next token, then drop the wrapper to get the token pushed
        // back on our buffer. Not the clearest solution, but gets around
        // the borrow checker.
        let _ = self.multipeek().peek_next()?;

        if let Some(&TokenOrPos::Tok(ref t)) = self.prev_buffered.last() {
            Some(t)
        } else {
            unreachable!("unexpected state: peeking next token failed. This is a bug!")
        }
    }
}

impl<I: Iterator<Item = Token>> Iterator for TokenIter<I> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        let mut ret = None;
        loop {
            // Make sure we update our current position before continuing.
            match self.next_token_or_pos() {
                Some(TokenOrPos::Tok(next)) => {
                    self.pos.advance(&next);
                    ret = Some(next);
                    break;
                }

                Some(TokenOrPos::Pos(_)) => panic!("unexpected state. This is a bug!"),
                None => break,
            }
        }

        // Make sure we update our position according to any trailing `Pos` points.
        // The parser expects that polling our current position will give it the
        // position of the next token we will yield. If we perform this check right
        // before yielding the next token, the parser will believe that token appears
        // much earlier in the source than it actually does.
        self.updated_buffered_pos();
        ret
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let (low_hint, hi) = self.iter.size_hint();
        let low = if self.prev_buffered.is_empty() {
            low_hint
        } else {
            self.prev_buffered.len()
        };
        (low, hi)
    }
}

impl<I: Iterator<Item = Token>> RewindableTokenIterator for TokenIter<I> {
    fn rewind(&mut self, tokens: Vec<TokenOrPos>) {
        self.buffer_tokens_and_positions_to_yield_first(tokens, None);
    }

    fn next_token_or_pos(&mut self) -> Option<TokenOrPos> {
        self.prev_buffered
            .pop()
            .or_else(|| self.iter.next().map(TokenOrPos::Tok))
    }
}

impl<I: Iterator<Item = Token>> TokenIterator for TokenIter<I> {}

impl<I: Iterator<Item = Token>> TokenIter<I> {
    /// Creates a new TokenIter from another Token iterator.
    pub fn new(iter: I) -> TokenIter<I> {
        TokenIter {
            iter: iter.fuse(),
            prev_buffered: Vec::new(),
            pos: SourcePos::new(),
        }
    }

    /// Creates a new TokenIter from another Token iterator and an initial position.
    pub fn with_position(iter: I, pos: SourcePos) -> TokenIter<I> {
        let mut iter = TokenIter::new(iter);
        iter.pos = pos;
        iter
    }

    /// Return a wrapper which allows for arbitrary look ahead. Dropping the
    /// wrapper will restore the internal stream back to what it was.
    pub fn multipeek(&mut self) -> Multipeek<'_> {
        Multipeek::new(self)
    }

    /// Update the current position based on any buffered state.
    ///
    /// This allows us to always correctly report the position of the next token
    /// we are about to yield.
    fn updated_buffered_pos(&mut self) {
        while let Some(&TokenOrPos::Pos(pos)) = self.prev_buffered.last() {
            self.prev_buffered.pop();
            self.pos = pos;
        }
    }

    /// Accepts a vector of tokens (and positions) to be yielded completely before the
    /// inner iterator is advanced further. The optional `buf_start` (if provided)
    /// indicates what the iterator's position should have been if we were to naturally
    /// yield the provided buffer.
    fn buffer_tokens_and_positions_to_yield_first(
        &mut self,
        mut tokens: Vec<TokenOrPos>,
        token_start: Option<SourcePos>,
    ) {
        self.prev_buffered.reserve(tokens.len() + 1);

        // Push the current position further up the stack since we want to
        // restore it before yielding any previously-peeked tokens.
        if token_start.is_some() {
            self.prev_buffered.push(TokenOrPos::Pos(self.pos));
        }

        // Buffer the newly provided tokens in reverse since we are using a stack
        tokens.reverse();
        self.prev_buffered.extend(tokens);

        // Set our position to what it should be as we yield the buffered tokens
        if let Some(p) = token_start {
            self.pos = p;
        }

        self.updated_buffered_pos();
    }

    /// Accepts a vector of tokens to be yielded completely before the inner
    /// iterator is advanced further. The provided `buf_start` indicates
    /// what the iterator's position should have been if we were to naturally
    /// yield the provided buffer.
    pub fn buffer_tokens_to_yield_first(&mut self, buf: Vec<Token>, buf_start: SourcePos) {
        let tokens = buf.into_iter().map(TokenOrPos::Tok).collect();
        self.buffer_tokens_and_positions_to_yield_first(tokens, Some(buf_start));
    }

    /// Collects all tokens yielded by `TokenIter::backticked_remove_backslashes`
    /// and creates a `TokenIter` which will yield the collected tokens, and maintain
    /// the correct position of where each token appears in the original source,
    /// regardless of how many backslashes may have been removed since then.
    pub fn token_iter_from_backticked_with_removed_backslashes(
        &mut self,
        pos: SourcePos,
    ) -> Result<TokenIter<std_iter::Empty<Token>>, UnmatchedError> {
        BacktickBackslashRemover::create_token_iter(self.backticked(pos))
    }
}

/// A wrapper for peeking arbitrary amounts into a `Token` stream.
/// Inspired by the `Multipeek` implementation in the `itertools` crate.
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct Multipeek<'a> {
    /// The underlying token iterator. This is pretty much just a `TokenIter`,
    /// but we use a trait object to avoid having a generic signature and
    /// make this wrapper more flexible.
    iter: &'a mut dyn RewindableTokenIterator,
    /// A buffer of values taken from the underlying iterator, in the order
    /// they were pulled.
    buf: Vec<TokenOrPos>,
}

impl<'a> Drop for Multipeek<'a> {
    fn drop(&mut self) {
        let tokens = mem::replace(&mut self.buf, Vec::new());
        self.iter.rewind(tokens);
    }
}

impl<'a> Multipeek<'a> {
    /// Wrap an iterator for arbitrary look-ahead.
    fn new(iter: &'a mut dyn RewindableTokenIterator) -> Self {
        Multipeek {
            iter,
            buf: Vec::new(),
        }
    }

    /// Public method for lazily peeking the next (unpeeked) value.
    /// Implemented as its own API instead of as an `Iterator` to avoid
    /// confusion with advancing the regular iterator.
    pub fn peek_next(&mut self) -> Option<&Token> {
        loop {
            match self.iter.next_token_or_pos() {
                Some(t) => {
                    let is_tok = t.is_tok();
                    self.buf.push(t);

                    if is_tok {
                        break;
                    }
                }
                None => return None,
            }
        }

        if let Some(&TokenOrPos::Tok(ref t)) = self.buf.last() {
            Some(t)
        } else {
            None
        }
    }
}

/// A wrapper which allows treating `TokenIter<I>` and `TokenIter<Empty<_>>` as
/// the same thing, even though they are technically different types.
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Debug)]
pub enum TokenIterWrapper<I> {
    /// A `TokenIter` which holds an aribtrary `Iterator` over `Token`s.
    Regular(TokenIter<I>),
    /// A `TokenIter` which has all `Token`s buffered in memory, and thus
    /// has no underlying iterator.
    Buffered(TokenIter<std_iter::Empty<Token>>),
}

impl<I: Iterator<Item = Token>> PositionIterator for TokenIterWrapper<I> {
    fn pos(&self) -> SourcePos {
        match *self {
            TokenIterWrapper::Regular(ref inner) => inner.pos(),
            TokenIterWrapper::Buffered(ref inner) => inner.pos(),
        }
    }
}

impl<I: Iterator<Item = Token>> PeekableIterator for TokenIterWrapper<I> {
    fn peek(&mut self) -> Option<&Self::Item> {
        match *self {
            TokenIterWrapper::Regular(ref mut inner) => inner.peek(),
            TokenIterWrapper::Buffered(ref mut inner) => inner.peek(),
        }
    }
}

impl<I: Iterator<Item = Token>> Iterator for TokenIterWrapper<I> {
    type Item = Token;

    fn next(&mut self) -> Option<Self::Item> {
        match *self {
            TokenIterWrapper::Regular(ref mut inner) => inner.next(),
            TokenIterWrapper::Buffered(ref mut inner) => inner.next(),
        }
    }
}

impl<I: Iterator<Item = Token>> TokenIterator for TokenIterWrapper<I> {}

impl<I: Iterator<Item = Token>> TokenIterWrapper<I> {
    /// Return a wrapper which allows for arbitrary look ahead. Dropping the
    /// wrapper will restore the internal stream back to what it was.
    pub fn multipeek(&mut self) -> Multipeek<'_> {
        match *self {
            TokenIterWrapper::Regular(ref mut inner) => inner.multipeek(),
            TokenIterWrapper::Buffered(ref mut inner) => inner.multipeek(),
        }
    }

    /// Delegates to `TokenIter::buffer_tokens_to_yield_first`.
    pub fn buffer_tokens_to_yield_first(&mut self, buf: Vec<Token>, buf_start: SourcePos) {
        match *self {
            TokenIterWrapper::Regular(ref mut inner) => {
                inner.buffer_tokens_to_yield_first(buf, buf_start)
            }
            TokenIterWrapper::Buffered(ref mut inner) => {
                inner.buffer_tokens_to_yield_first(buf, buf_start)
            }
        }
    }

    /// Delegates to `TokenIter::token_iter_from_backticked_with_removed_backslashes`.
    pub fn token_iter_from_backticked_with_removed_backslashes(
        &mut self,
        pos: SourcePos,
    ) -> Result<TokenIter<std_iter::Empty<Token>>, UnmatchedError> {
        match *self {
            TokenIterWrapper::Regular(ref mut inner) => {
                inner.token_iter_from_backticked_with_removed_backslashes(pos)
            }
            TokenIterWrapper::Buffered(ref mut inner) => {
                inner.token_iter_from_backticked_with_removed_backslashes(pos)
            }
        }
    }
}

/// An iterator that yields at least one token, but continues to yield
/// tokens until all matching cases of single/double quotes, backticks,
/// ${ }, $( ), or ( ) are found.
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Debug)]
pub struct Balanced<I> {
    /// The underlying token iterator.
    iter: I,
    /// Any token we had to peek after a backslash but haven't yielded yet,
    /// as well as the position after it.
    escaped: Option<(Token, SourcePos)>,
    /// A stack of pending unmatched tokens we still must encounter.
    stack: Vec<(Token, SourcePos)>,
    /// Indicates if we should yield the final, outermost delimeter.
    skip_last_delimeter: bool,
    /// Makes the iterator *fused* by yielding None forever after we are done.
    done: bool,
    /// The current position of the iterator.
    pos: SourcePos,
}

impl<I: PositionIterator> Balanced<I> {
    /// Constructs a new balanced iterator.
    ///
    /// If no delimeter is given, a single token will be yielded, unless the
    /// first found token is an opening one (e.g. "), making the iterator yield
    /// tokens until its matching delimeter is found (the matching delimeter *will*
    /// be consumed).
    ///
    /// If a delimeter (and its position) is specified, tokens are yielded *up to*
    /// the delimeter, but the delimeter will be silently consumed.
    pub fn new(iter: I, delim: Option<(Token, SourcePos)>) -> Self {
        Balanced {
            escaped: None,
            skip_last_delimeter: delim.is_some(),
            stack: delim.map_or(Vec::new(), |d| vec![d]),
            done: false,
            pos: iter.pos(),
            iter,
        }
    }
}

impl<I: PeekablePositionIterator<Item = Token>> PositionIterator for Balanced<I> {
    fn pos(&self) -> SourcePos {
        self.pos
    }
}

impl<I: PeekablePositionIterator<Item = Token>> Iterator for Balanced<I> {
    type Item = Result<Token, UnmatchedError>;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some((tok, pos)) = self.escaped.take() {
            self.pos = pos;
            return Some(Ok(tok));
        } else if self.done {
            return None;
        }

        if self.stack.last().map(|t| &t.0) == self.iter.peek() {
            let ret = self.iter.next().map(Ok);
            self.stack.pop();
            let stack_empty = self.stack.is_empty();
            self.done |= stack_empty;
            self.pos = self.iter.pos();
            if self.skip_last_delimeter && stack_empty {
                return None;
            } else {
                return ret;
            };
        }

        // Tokens between single quotes have no special meaning
        // so we should make sure we don't treat anything specially.
        if let Some(&(SingleQuote, pos)) = self.stack.last() {
            let ret = match self.iter.next() {
                // Closing SingleQuote should have been captured above
                Some(t) => Some(Ok(t)),
                // Make sure we indicate errors on missing closing quotes
                None => Some(Err(UnmatchedError(SingleQuote, pos))),
            };

            self.pos = self.iter.pos();
            return ret;
        }

        let cur_pos = self.iter.pos();
        let ret = match self.iter.next() {
            Some(Backslash) => {
                // Make sure that we indicate our position as before the escaped token,
                // and NOT as the underlying iterator's position, which will indicate the
                // position AFTER the escaped token (which we are buffering ourselves)
                self.pos = self.iter.pos();

                debug_assert_eq!(self.escaped, None);
                self.escaped = self.iter.next().map(|t| (t, self.iter.pos()));
                // Make sure we stop yielding tokens after the stored escaped token
                // otherwise we risk consuming one token too many!
                self.done |= self.stack.is_empty();
                return Some(Ok(Backslash));
            }

            Some(Backtick) => {
                self.stack.push((Backtick, cur_pos));
                Some(Ok(Backtick))
            }

            Some(SingleQuote) => {
                if self.stack.last().map(|t| &t.0) != Some(&DoubleQuote) {
                    self.stack.push((SingleQuote, cur_pos));
                }
                Some(Ok(SingleQuote))
            }

            Some(DoubleQuote) => {
                self.stack.push((DoubleQuote, cur_pos));
                Some(Ok(DoubleQuote))
            }

            Some(ParenOpen) => {
                self.stack.push((ParenClose, cur_pos));
                Some(Ok(ParenOpen))
            }

            Some(Dollar) => {
                let cur_pos = self.iter.pos(); // Want the pos of curly or paren, not $ here
                match self.iter.peek() {
                    Some(&CurlyOpen) => self.stack.push((CurlyClose, cur_pos)),
                    Some(&ParenOpen) => {} // Already handled by paren case above

                    // We have nothing further to match
                    _ => {
                        self.done |= self.stack.is_empty();
                    }
                }
                Some(Ok(Dollar))
            }

            Some(t) => {
                // If we aren't looking for any more delimeters we should only
                // consume a single token (since its balanced by nature)
                self.done |= self.stack.is_empty();
                Some(Ok(t))
            }

            None => match self.stack.pop() {
                // Its okay to hit EOF if everything is balanced so far
                None => {
                    self.done = true;
                    None
                }
                // But its not okay otherwise
                Some((ParenClose, pos)) => Some(Err(UnmatchedError(ParenOpen, pos))),
                Some((CurlyClose, pos)) => Some(Err(UnmatchedError(CurlyOpen, pos))),
                Some((delim, pos)) => Some(Err(UnmatchedError(delim, pos))),
            },
        };

        self.pos = self.iter.pos();
        ret
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        // Our best guess is as good as the internal token iterator's...
        self.iter.size_hint()
    }
}

/// A `Balanced` backtick `Token` iterator which removes all backslashes
/// from the stream that are followed by \, $, or `.
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Debug)]
pub struct BacktickBackslashRemover<I> {
    /// The underlying token iterator.
    iter: Balanced<I>,
    peeked: Option<Result<Token, UnmatchedError>>,
    /// Makes the iterator *fused* by yielding None forever after we are done.
    done: bool,
}

impl<I> BacktickBackslashRemover<I> {
    /// Constructs a new balanced backtick iterator which removes all backslashes
    /// from the stream that are followed by \, $, or `.
    pub fn new(iter: Balanced<I>) -> Self {
        BacktickBackslashRemover {
            iter,
            peeked: None,
            done: false,
        }
    }
}

impl<I: PeekablePositionIterator<Item = Token>> BacktickBackslashRemover<I> {
    /// Collects all tokens yielded by `TokenIter::backticked_remove_backslashes`
    /// and creates a `TokenIter` which will yield the collected tokens, and maintain
    /// the correct position of where each token appears in the original source,
    /// regardless of how many backslashes may have been removed since then.
    fn create_token_iter(
        mut iter: Balanced<I>,
    ) -> Result<TokenIter<std_iter::Empty<Token>>, UnmatchedError> {
        let mut all_chunks = Vec::new();
        let mut chunk_start = iter.pos();
        let mut chunk = Vec::new();

        loop {
            match iter.next() {
                Some(Ok(Backslash)) => {
                    let next_pos = iter.pos();
                    match iter.next() {
                        Some(Ok(tok @ Dollar))
                        | Some(Ok(tok @ Backtick))
                        | Some(Ok(tok @ Backslash)) => {
                            all_chunks.push((chunk, chunk_start));
                            chunk_start = next_pos;
                            chunk = vec![tok];
                        }

                        Some(tok) => {
                            chunk.push(Backslash);
                            chunk.push(tok?);
                        }

                        None => chunk.push(Backslash),
                    }
                }

                Some(tok) => chunk.push(tok?),
                None => break,
            }
        }

        if !chunk.is_empty() {
            all_chunks.push((chunk, chunk_start));
        }

        let mut tok_iter = TokenIter::with_position(std_iter::empty(), iter.pos());
        while let Some((chunk, chunk_end)) = all_chunks.pop() {
            tok_iter.buffer_tokens_to_yield_first(chunk, chunk_end);
        }
        Ok(tok_iter)
    }
}

impl<I> std::iter::FusedIterator for BacktickBackslashRemover<I> where
    I: PeekablePositionIterator<Item = Token>
{
}

impl<I: PeekablePositionIterator<Item = Token>> Iterator for BacktickBackslashRemover<I> {
    type Item = Result<Token, UnmatchedError>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.peeked.is_some() {
            return self.peeked.take();
        } else if self.done {
            return None;
        }

        match self.iter.next() {
            Some(Ok(Backslash)) => match self.iter.next() {
                ret @ Some(Ok(Dollar)) | ret @ Some(Ok(Backtick)) | ret @ Some(Ok(Backslash)) => {
                    ret
                }

                Some(t) => {
                    debug_assert!(self.peeked.is_none());
                    self.peeked = Some(t);
                    Some(Ok(Backslash))
                }

                None => {
                    self.done = true;
                    Some(Ok(Backslash))
                }
            },

            Some(t) => Some(t),
            None => {
                self.done = true;
                None
            }
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        // The number of tokens we actually yield will never be
        // more than those of the underlying iterator, and will
        // probably be less, but this is a good enough estimate.
        self.iter.size_hint()
    }
}

#[cfg(test)]
mod tests {
    use super::{PositionIterator, TokenIter, TokenOrPos};
    use crate::parse::SourcePos;
    use crate::token::Token;

    #[test]
    fn test_multipeek() {
        let tokens = vec![
            Token::ParenOpen,
            Token::Semi,
            Token::Dollar,
            Token::ParenClose,
        ];

        let mut tok_iter = TokenIter::new(tokens.clone().into_iter());
        {
            let mut multipeek = tok_iter.multipeek();
            let mut expected_peeked = tokens.iter();
            while let Some(t) = multipeek.peek_next() {
                assert_eq!(expected_peeked.next(), Some(t));
            }

            // Exhausted the expected stream
            assert_eq!(expected_peeked.next(), None);
        }

        // Original iterator still yields the expected values
        assert_eq!(tokens, tok_iter.collect::<Vec<_>>());
    }

    #[test]
    fn test_buffering_tokens_should_immediately_update_position() {
        fn src(byte: usize, line: usize, col: usize) -> SourcePos {
            SourcePos { byte, line, col }
        }

        let mut tok_iter = TokenIter::new(std::iter::empty());

        let pos = src(4, 4, 4);

        tok_iter.buffer_tokens_and_positions_to_yield_first(
            vec![
                TokenOrPos::Pos(src(2, 2, 2)),
                TokenOrPos::Pos(src(3, 3, 3)),
                TokenOrPos::Pos(pos),
                TokenOrPos::Tok(Token::Newline),
            ],
            Some(src(1, 1, 1)),
        );

        assert_eq!(tok_iter.pos(), pos);
    }
}


#![deny(rust_2018_idioms)]
use conch_parser::ast::PipeableCommand::*;
use conch_parser::ast::*;
use conch_parser::parse::ParseError::*;
use conch_parser::token::Token;

mod parse_support;
use crate::parse_support::*;

#[test]
fn test_and_or_correct_associativity() {
    let mut p = make_parser("foo || bar && baz");
    let correct = CommandList {
        first: ListableCommand::Single(Simple(cmd_simple("foo"))),
        rest: vec![
            AndOr::Or(ListableCommand::Single(Simple(cmd_simple("bar")))),
            AndOr::And(ListableCommand::Single(Simple(cmd_simple("baz")))),
        ],
    };
    assert_eq!(correct, p.and_or_list().unwrap());
}

#[test]
fn test_and_or_valid_with_newlines_after_operator() {
    let mut p = make_parser("foo ||\n\n\n\nbar && baz");
    let correct = CommandList {
        first: ListableCommand::Single(Simple(cmd_simple("foo"))),
        rest: vec![
            AndOr::Or(ListableCommand::Single(Simple(cmd_simple("bar")))),
            AndOr::And(ListableCommand::Single(Simple(cmd_simple("baz")))),
        ],
    };
    assert_eq!(correct, p.and_or_list().unwrap());
}

#[test]
fn test_and_or_invalid_with_newlines_before_operator() {
    let mut p = make_parser("foo || bar\n\n&& baz");
    p.and_or_list().unwrap(); // Successful parse Or(foo, bar)
                              // Fail to parse "&& baz" which is an error
    assert_eq!(
        Err(Unexpected(Token::AndIf, src(12, 3, 1))),
        p.complete_command()
    );
}