/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! Methods for the `list` type.

use starlark_derive::starlark_module;

use crate as starlark;
use crate::environment::MethodsBuilder;
use crate::stdlib::util::convert_index;
use crate::stdlib::util::convert_indices;
use crate::values::list::ListRef;
use crate::values::none::NoneOr;
use crate::values::none::NoneType;
use crate::values::types::list::value::ListData;
use crate::values::Heap;
use crate::values::StarlarkIter;
use crate::values::Value;
use crate::values::ValueError;
use crate::values::ValueOfUnchecked;

#[starlark_module]
pub(crate) fn list_methods(builder: &mut MethodsBuilder) {
    /// [list.append](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·append
    /// ): append an element to a list.
    ///
    /// `L.append(x)` appends `x` to the list L, and returns `None`.
    ///
    /// `append` fails if the list is frozen or has active iterators.
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = []
    /// x.append(1)
    /// x.append(2)
    /// x.append(3)
    /// x == [1, 2, 3]
    /// # "#);
    /// ```
    fn append<'v>(
        this: Value<'v>,
        #[starlark(require = pos)] el: Value<'v>,
        heap: &'v Heap,
    ) -> anyhow::Result<NoneType> {
        let this = ListData::from_value_mut(this)?;
        this.push(el, heap);
        Ok(NoneType)
    }

    /// [list.clear](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·clear
    /// ): clear a list
    ///
    /// `L.clear()` removes all the elements of the list L and returns `None`.
    /// It fails if the list is frozen or if there are active iterators.
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = [1, 2, 3]
    /// x.clear()
    /// x == []
    /// # "#);
    /// ```
    fn clear(this: Value) -> anyhow::Result<NoneType> {
        let this = ListData::from_value_mut(this)?;
        this.clear();
        Ok(NoneType)
    }

    /// [list.extend](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·extend
    /// ): extend a list with another iterable's content.
    ///
    /// `L.extend(x)` appends the elements of `x`, which must be iterable, to
    /// the list L, and returns `None`.
    ///
    /// `extend` fails if `x` is not iterable, or if the list L is frozen or has
    /// active iterators.
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = []
    /// x.extend([1, 2, 3])
    /// x.extend(["foo"])
    /// x == [1, 2, 3, "foo"]
    /// # "#);
    /// ```
    fn extend<'v>(
        this: Value<'v>,
        #[starlark(require = pos)] other: ValueOfUnchecked<'v, StarlarkIter<Value<'v>>>,
        heap: &'v Heap,
    ) -> anyhow::Result<NoneType> {
        let res = ListData::from_value_mut(this)?;
        if this.ptr_eq(other.get()) {
            // If the types alias, we can't borrow the `other` for iteration.
            // But we can do something smarter to double the elements
            res.double(heap);
        } else {
            let it = other.get().iterate(heap)?;
            res.extend(it, heap);
        }
        Ok(NoneType)
    }

    /// [list.index](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·index
    /// ): get the index of an element in the list.
    ///
    /// `L.index(x[, start[, end]])` finds `x` within the list L and returns its
    /// index.
    ///
    /// The optional `start` and `end` parameters restrict the portion of
    /// list L that is inspected.  If provided and not `None`, they must be list
    /// indices of type `int`. If an index is negative, `len(L)` is effectively
    /// added to it, then if the index is outside the range `[0:len(L)]`, the
    /// nearest value within that range is used; see [Indexing](#indexing).
    ///
    /// `index` fails if `x` is not found in L, or if `start` or `end`
    /// is not a valid index (`int` or `None`).
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = ["b", "a", "n", "a", "n", "a"]
    /// # (
    /// x.index("a") == 1      # bAnana
    /// # and
    /// x.index("a", 2) == 3   # banAna
    /// # and
    /// x.index("a", -2) == 5  # bananA
    /// # )"#);
    /// ```
    #[starlark(speculative_exec_safe)]
    fn index<'v>(
        this: &ListRef<'v>,
        #[starlark(require = pos)] needle: Value<'v>,
        #[starlark(require = pos, default = NoneOr::None)] start: NoneOr<i32>,
        #[starlark(require = pos, default = NoneOr::None)] end: NoneOr<i32>,
    ) -> anyhow::Result<i32> {
        let (start, end) = convert_indices(this.len() as i32, start, end);
        if let Some(haystack) = this.get(start..end) {
            for (i, x) in haystack.iter().enumerate() {
                if x.equals(needle)? {
                    return Ok((i + start) as i32);
                }
            }
        }
        Err(anyhow::anyhow!(
            "Element '{}' not found in '{}'",
            needle,
            this
        ))
    }

    /// [list.insert](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·insert
    /// ): insert an element in a list.
    ///
    /// `L.insert(i, x)` inserts the value `x` in the list L at index `i`,
    /// moving higher-numbered elements along by one.  It returns `None`.
    ///
    /// As usual, the index `i` must be an `int`. If its value is negative,
    /// the length of the list is added, then its value is clamped to the
    /// nearest value in the range `[0:len(L)]` to yield the effective index.
    ///
    /// `insert` fails if the list is frozen or has active iterators.
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = ["b", "c", "e"]
    /// x.insert(0, "a")
    /// x.insert(-1, "d")
    /// x == ["a", "b", "c", "d", "e"]
    /// # "#);
    /// ```
    fn insert<'v>(
        this: Value<'v>,
        #[starlark(require = pos)] index: i32,
        #[starlark(require = pos)] el: Value<'v>,
        heap: &'v Heap,
    ) -> anyhow::Result<NoneType> {
        let this = ListData::from_value_mut(this)?;
        let index = convert_index(this.len() as i32, index);
        this.insert(index, el, heap);
        Ok(NoneType)
    }

    /// [list.pop](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·pop
    /// ): removes and returns the last element of a list.
    ///
    /// `L.pop([index])` removes and returns the last element of the list L, or,
    /// if the optional index is provided, at that index.
    ///
    /// `pop` fails if the index is negative or not less than the length of
    /// the list, of if the list is frozen or has active iterators.
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = [1, 2, 3]
    /// # (
    /// x.pop() == 3
    /// # and
    /// x.pop() == 2
    /// # and
    /// x == [1]
    /// # )"#);
    /// ```
    fn pop<'v>(
        this: Value<'v>,
        #[starlark(require = pos)] index: Option<i32>,
    ) -> anyhow::Result<Value<'v>> {
        let this = ListData::from_value_mut(this)?;
        let index = index.unwrap_or_else(|| (this.len() as i32) - 1);
        if index < 0 || index >= this.len() as i32 {
            return Err(ValueError::IndexOutOfBound(index).into());
        }
        Ok(this.remove(index as usize))
    }

    /// [list.remove](
    /// https://github.com/google/skylark/blob/3705afa472e466b8b061cce44b47c9ddc6db696d/doc/spec.md#listÂ·remove
    /// ): remove a value from a list
    ///
    /// `L.remove(x)` removes the first occurrence of the value `x` from the
    /// list L, and returns `None`.
    ///
    /// `remove` fails if the list does not contain `x`, is frozen, or has
    /// active iterators.
    ///
    /// ```
    /// # starlark::assert::is_true(r#"
    /// x = [1, 2, 3, 2]
    /// x.remove(2)
    /// # t = (
    /// x == [1, 3, 2]
    /// # )
    /// x.remove(2)
    /// # (t and (
    /// x == [1, 3]
    /// # ))"#);
    /// ```
    ///
    /// A subsequent call to `x.remove(2)` would yield an error because the
    /// element won't be found.
    ///
    /// ```
    /// # starlark::assert::fail(r#"
    /// x = [1, 2, 3, 2]
    /// x.remove(2)
    /// x.remove(2)
    /// x.remove(2) # error: not found
    /// # "#, "not found");
    /// ```
    fn remove<'v>(
        this: Value<'v>,
        #[starlark(require = pos)] needle: Value<'v>,
    ) -> anyhow::Result<NoneType> {
        // Written in two separate blocks so we ensure we give up the
        // immutable borrow before making the mutable borrow.
        let position = {
            let this = ListRef::from_value(this).unwrap();
            let position = this.iter().position(|v| v == needle);
            match position {
                Some(i) => i,
                None => {
                    return Err(anyhow::anyhow!(
                        "Element '{}' not found in list '{}'",
                        needle,
                        this
                    ));
                }
            }
        };
        {
            // now mutate it with no further value calls
            let this = ListData::from_value_mut(this)?;
            this.remove(position);
            Ok(NoneType)
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::assert;

    #[test]
    fn test_error_codes() {
        assert::fail(
            "x = [1, 2, 3, 2]; x.remove(2); x.remove(2); x.remove(2)",
            "not found in list",
        );
    }

    #[test]
    fn test_index() {
        // Should fail, but should not panic.
        assert::fail("[True].index(True, 1, 0)", "not found");
    }

    #[test]
    fn recursive_list() {
        assert::is_true(
            r#"
cyclic = [1, 2, 3]
cyclic[1] = cyclic
len(cyclic) == 3 and len(cyclic[1]) == 3
    "#,
        )
    }
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! A module with the standard function and constants that are by default in all
//! dialect of Starlark

use dupe::Dupe;

use crate::environment::GlobalsBuilder;

pub(crate) mod breakpoint;
pub(crate) mod dict;
pub(crate) mod enumeration;
pub(crate) mod extra;
mod funcs;
pub(crate) mod json;
pub(crate) mod partial;

pub(crate) mod list;
pub(crate) mod string;
pub(crate) mod structs;
pub(crate) mod util;

pub use extra::PrintHandler;

use crate::values::record::globals::register_record;
use crate::values::typing;

/// Return the default global environment, it is not yet frozen so that a caller
/// can refine it.
///
/// For example `stdlib::standard_environment().freeze().child("test")` create a
/// child environment of this global environment that have been frozen.
pub(crate) fn standard_environment() -> GlobalsBuilder {
    GlobalsBuilder::new().with(funcs::global_functions)
}

/// The extra library definitions available in this Starlark implementation, but not in the standard.
#[derive(PartialEq, Eq, Copy, Clone, Dupe)]
pub enum LibraryExtension {
    /// Definitions to support the `struct` type, the `struct()` constructor.
    StructType,
    /// Definitions to support the `record` type, the `record()` constructor and `field()` function.
    RecordType,
    /// Definitions to support the `enum` type, the `enum()` constructor.
    EnumType,
    /// A function `map(f, xs)` which applies `f` to each element of `xs` and returns the result.
    Map,
    /// A function `filter(f, xs)` which applies `f` to each element of `xs` and returns those for which `f` returns `True`.
    /// As a special case, `filter(None, xs)` removes all `None` values.
    Filter,
    /// Partially apply a function, `partial(f, *args, **kwargs)` will create a function where those `args` `kwargs`
    /// are already applied to `f`.
    Partial,
    /// Create a regex from a string.
    ExperimentalRegex,
    /// Add a function `debug(x)` which shows the Rust [`Debug`](std::fmt::Debug) representation of a value.
    /// Useful when debugging, but the output should not be considered stable.
    Debug,
    /// Add a function `print(x)` which prints to stderr.
    Print,
    /// Add a function `pprint(x)` which pretty-prints to stderr.
    Pprint,
    /// Add a function `breakpoint()` which will drop into a console-module evaluation prompt.
    Breakpoint,
    /// Add a function `json()` which will generate JSON for a module.
    Json,
    /// Add a function `abs()` which will take the absolute value of an int.
    Abs,
    /// `type_compiled()` function.
    Typing,
    // Make sure if you add anything new, you add it to `all` below.
}

impl LibraryExtension {
    /// A list of all extensions that will be updated as new methods are added.
    pub fn all() -> &'static [Self] {
        use LibraryExtension::*;
        &[
            StructType,
            RecordType,
            EnumType,
            Map,
            Filter,
            Partial,
            ExperimentalRegex,
            Debug,
            Print,
            Pprint,
            Breakpoint,
            Json,
            Abs,
            Typing,
        ]
    }

    /// Add a specific extension to a [`GlobalsBuilder`].
    pub fn add(self, builder: &mut GlobalsBuilder) {
        use LibraryExtension::*;
        match self {
            StructType => structs::global(builder),
            RecordType => register_record(builder),
            EnumType => enumeration::global(builder),
            Map => extra::map(builder),
            Filter => extra::filter(builder),
            Partial => partial::partial(builder),
            ExperimentalRegex => extra::regex(builder),
            Debug => extra::debug(builder),
            Print => extra::print(builder),
            Pprint => extra::pprint(builder),
            Breakpoint => breakpoint::global(builder),
            Json => json::json(builder),
            Abs => extra::abs(builder),
            Typing => typing::register_eval_type(builder),
        }
    }
}

#[cfg(test)]
mod tests {
    use allocative::Allocative;
    use derive_more::Display;
    use dupe::Dupe;
    use starlark_derive::starlark_module;
    use starlark_derive::starlark_value;
    use starlark_derive::NoSerialize;

    use crate as starlark;
    use crate::any::ProvidesStaticType;
    use crate::assert::Assert;
    use crate::environment::GlobalsBuilder;
    use crate::environment::Methods;
    use crate::environment::MethodsBuilder;
    use crate::environment::MethodsStatic;
    use crate::starlark_simple_value;
    use crate::values::none::NoneType;
    use crate::values::StarlarkValue;
    use crate::values::UnpackValue;
    use crate::values::Value;
    use crate::values::ValueLike;

    #[test]
    fn test_no_arg() {
        #[starlark_module]
        fn global(builder: &mut GlobalsBuilder) {
            fn nop() -> anyhow::Result<NoneType> {
                Ok(NoneType)
            }
        }

        let env = GlobalsBuilder::new().with(global).build();
        env.get("nop").unwrap();
    }

    #[test]
    fn test_value_attributes() {
        #[derive(
            Copy,
            Clone,
            Debug,
            Dupe,
            PartialEq,
            Display,
            ProvidesStaticType,
            NoSerialize,
            Allocative
        )]
        struct Bool2(bool);
        starlark_simple_value!(Bool2);

        #[starlark_value(type = "bool2")]
        impl<'v> StarlarkValue<'v> for Bool2 {
            fn get_methods() -> Option<&'static Methods> {
                static RES: MethodsStatic = MethodsStatic::new();
                RES.methods(methods)
            }

            fn equals(&self, other: Value<'v>) -> anyhow::Result<bool> {
                match other.downcast_ref::<Bool2>() {
                    None => Ok(false),
                    Some(v) => Ok(*v == *self),
                }
            }
        }

        impl<'v> UnpackValue<'v> for Bool2 {
            fn unpack_value(value: Value<'v>) -> Option<Self> {
                Some(*value.downcast_ref::<Bool2>().unwrap())
            }
        }

        #[starlark_module]
        fn globals(builder: &mut GlobalsBuilder) {
            const True2: Bool2 = Bool2(true);
            const False2: Bool2 = Bool2(false);
        }

        #[starlark_module]
        fn methods(builder: &mut MethodsBuilder) {
            #[starlark(attribute)]
            fn invert1(this: Bool2) -> anyhow::Result<Bool2> {
                Ok(Bool2(!this.0))
            }

            fn invert2(this: Bool2) -> anyhow::Result<Bool2> {
                Ok(Bool2(!this.0))
            }
        }

        let mut a = Assert::new();
        a.globals_add(globals);
        a.all_true(
            r#"
True2 == True2
True2 != False2
True2.invert1 == False2
False2.invert1 == True2
False2.invert2() == True2
hasattr(True2, "invert1") == True
hasattr(True2, "invert2") == True
hasattr(True2, "invert3") == False
dir(False2) == ["invert1","invert2"]
getattr(False2, "invert1") == True2
getattr(True2, "invert1") == False2
getattr(True2, "invert2")() == False2
"#,
        );
    }
}


/*
 * Copyright 2019 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use std::fmt;
use std::fmt::Display;

use allocative::Allocative;
use starlark_derive::starlark_module;
use starlark_derive::starlark_value;
use starlark_derive::NoSerialize;

use crate as starlark;
use crate::any::ProvidesStaticType;
use crate::coerce::coerce;
use crate::coerce::Coerce;
use crate::collections::symbol_map::Symbol;
use crate::environment::GlobalsBuilder;
use crate::eval::runtime::arguments::ArgNames;
use crate::eval::runtime::arguments::ArgumentsFull;
use crate::eval::runtime::rust_loc::rust_loc;
use crate::eval::Arguments;
use crate::eval::Evaluator;
use crate::slice_vec_ext::SliceExt;
use crate::slice_vec_ext::VecExt;
use crate::starlark_complex_values;
use crate::values::dict::DictRef;
use crate::values::function::FUNCTION_TYPE;
use crate::values::layout::typed::string::StringValueLike;
use crate::values::types::tuple::value::Tuple;
use crate::values::Freeze;
use crate::values::Freezer;
use crate::values::FrozenStringValue;
use crate::values::FrozenValue;
use crate::values::StarlarkValue;
use crate::values::StringValue;
use crate::values::Trace;
use crate::values::Value;
use crate::values::ValueLike;

#[starlark_module]
pub fn partial(builder: &mut GlobalsBuilder) {
    /// Construct a partial application. In almost all cases it is simpler to use a `lamdba`.
    fn partial<'v>(
        #[starlark(require = pos)] func: Value<'v>,
        #[starlark(args)] args: Value<'v>,
        #[starlark(kwargs)] kwargs: DictRef<'v>,
    ) -> anyhow::Result<Partial<'v>> {
        debug_assert!(Tuple::from_value(args).is_some());
        let names = kwargs
            .keys()
            .map(|x| {
                let x = StringValue::new(x).unwrap();
                (
                    // We duplicate string here.
                    // If this becomes hot, we should do better.
                    Symbol::new_hashed(x.as_str_hashed()),
                    x,
                )
            })
            .collect();
        Ok(Partial {
            func,
            pos: args,
            named: kwargs.values().collect(),
            names,
        })
    }
}

#[derive(Debug, Coerce, Trace, NoSerialize, ProvidesStaticType, Allocative)]
#[repr(C)]
struct PartialGen<V, S> {
    func: V,
    // Always references a tuple.
    pos: V,
    named: Vec<V>,
    names: Vec<(Symbol, S)>,
}

impl<'v, V: ValueLike<'v>, S> PartialGen<V, S> {
    fn pos_content(&self) -> &'v [Value<'v>] {
        Tuple::from_value(self.pos.to_value()).unwrap().content()
    }
}

impl<'v, V: ValueLike<'v>, S> Display for PartialGen<V, S> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "partial({}, *[", self.func)?;
        for (i, v) in self.pos_content().iter().enumerate() {
            if i != 0 {
                write!(f, ",")?;
            }
            v.fmt(f)?;
        }
        write!(f, "], **{{")?;
        for (i, (k, v)) in self.names.iter().zip(self.named.iter()).enumerate() {
            if i != 0 {
                write!(f, ",")?;
            }
            write!(f, "{}:", k.0.as_str())?;
            v.to_value().fmt(f)?;
        }
        write!(f, "}})")
    }
}

type Partial<'v> = PartialGen<Value<'v>, StringValue<'v>>;
type FrozenPartial = PartialGen<FrozenValue, FrozenStringValue>;
starlark_complex_values!(Partial);

impl<'v> Freeze for Partial<'v> {
    type Frozen = FrozenPartial;
    fn freeze(self, freezer: &Freezer) -> anyhow::Result<Self::Frozen> {
        Ok(FrozenPartial {
            func: self.func.freeze(freezer)?,
            pos: freezer.freeze(self.pos)?,
            named: self.named.try_map(|x| x.freeze(freezer))?,
            names: self
                .names
                .into_try_map(|(s, x)| anyhow::Ok((s, x.freeze(freezer)?)))?,
        })
    }
}

#[starlark_value(type = FUNCTION_TYPE)]
impl<'v, V: ValueLike<'v> + 'v, S: StringValueLike<'v> + 'v> StarlarkValue<'v> for PartialGen<V, S>
where
    Self: ProvidesStaticType<'v>,
{
    fn name_for_call_stack(&self, _me: Value<'v>) -> String {
        "partial".to_owned()
    }

    fn invoke(
        &self,
        _me: Value<'v>,
        args: &Arguments<'v, '_>,
        eval: &mut Evaluator<'v, '_>,
    ) -> anyhow::Result<Value<'v>> {
        // apply the partial arguments first, then the remaining arguments I was given

        let self_pos = self.pos_content();
        let self_named = coerce(&self.named);
        let self_names = coerce(&self.names);

        eval.alloca_concat(self_pos, args.0.pos, |pos, eval| {
            eval.alloca_concat(self_named, args.0.named, |named, eval| {
                eval.alloca_concat(self_names, args.0.names.names(), |names, eval| {
                    let params = Arguments(ArgumentsFull {
                        pos,
                        named,
                        names: ArgNames::new(names),
                        args: args.0.args,
                        kwargs: args.0.kwargs,
                    });
                    self.func
                        .to_value()
                        .invoke_with_loc(Some(rust_loc!()), &params, eval)
                })
            })
        })
    }
}

#[cfg(test)]
mod tests {
    use crate::assert;

    fn eq(expected: &str, expr: &str) {
        let sum = r#"
def sum(a, b, *args, **kwargs):
    # print("a=%s b=%s args=%s kwargs=%s" % (a, b, args, kwargs))
    # TODO(nga): fix typecheck.
    args = noop((a, b)) + args
    return [args, kwargs]
"#;

        assert::eq(expected, &format!("{}{}", sum, expr));
    }

    #[test]
    fn test_simple() {
        eq(
            "[(1, 2, 3), {'other': True, 'third': None}]",
            "(partial(sum, 1, other=True))(2, 3, third=None)",
        );
    }

    #[test]
    fn test_star_to_partial() {
        eq(
            "[(1, 2, 3), {'other': True, 'third': None}]",
            "(partial(sum, *[1], **{'other': True}))(2, 3, third=None)",
        );
    }

    #[test]
    fn test_start_to_returned_func() {
        eq(
            "[(1, 2, 3), {'other': True, 'third': None}]",
            "(partial(sum, other=True))(*[1, 2, 3], **{'third': None})",
        );
    }

    #[test]
    fn test_no_args_to_partial() {
        eq(
            "[(1, 2, 3), {'other': True, 'third': None}]",
            "(partial(sum))(1, 2, 3, third=None, **{'other': True})",
        );
    }
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! Implementation of `struct` function.
use std::collections::BTreeMap;

use allocative::Allocative;
use dupe::Dupe;
use starlark_derive::starlark_module;

use crate as starlark;
use crate::codemap::Span;
use crate::codemap::Spanned;
use crate::environment::GlobalsBuilder;
use crate::eval::Arguments;
use crate::typing::error::TypingError;
use crate::typing::function::Arg;
use crate::typing::function::TyCustomFunctionImpl;
use crate::typing::oracle::ctx::TypingOracleCtx;
use crate::typing::structs::TyStruct;
use crate::typing::Ty;
use crate::values::structs::value::FrozenStruct;
use crate::values::structs::value::Struct;
use crate::values::Heap;

#[derive(Allocative, Clone, Copy, Dupe, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct StructType;

impl TyCustomFunctionImpl for StructType {
    fn validate_call(
        &self,
        _span: Span,
        args: &[Spanned<Arg>],
        oracle: TypingOracleCtx,
    ) -> Result<Ty, TypingError> {
        let mut fields = BTreeMap::new();
        let mut extra = false;
        for x in args {
            match &x.node {
                Arg::Pos(_) => {
                    return Err(oracle.msg_error(x.span, "Positional arguments not allowed"));
                }
                Arg::Args(_) => {
                    // Args can be empty, and this is valid call:
                    // ```
                    // struct(*[], **{})
                    // ```
                }
                Arg::Name(name, val) => {
                    fields.insert(name.clone(), val.clone());
                }
                Arg::Kwargs(_) => extra = true,
            }
        }
        Ok(Ty::custom(TyStruct { fields, extra }))
    }
}

#[starlark_module]
pub fn global(builder: &mut GlobalsBuilder) {
    #[starlark(
        ty_custom_function = StructType,
        as_type = FrozenStruct,
    )]
    fn r#struct<'v>(args: &Arguments<'v, '_>, heap: &'v Heap) -> anyhow::Result<Struct<'v>> {
        args.no_positional_args(heap)?;
        // TODO(nga): missing optimization: practically most `struct` invocations are
        //   performed with fixed named arguments, e.g. `struct(a = 1, b = 2)`.
        //   In this case we can avoid allocating the map, but instead
        //   allocate field index once at compilation time and store field values in a vector.
        Ok(Struct::new(args.names_map()?))
    }
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use crate::values::none::NoneOr;

fn bound(val: i32, limit: i32) -> usize {
    if val <= 0 {
        0
    } else if val >= limit {
        limit as usize
    } else {
        val as usize
    }
}

pub(crate) fn convert_indices(len: i32, start: NoneOr<i32>, end: NoneOr<i32>) -> (usize, usize) {
    let start = start.into_option().unwrap_or(0);
    let end = end.into_option().unwrap_or(len);
    let end = if end < 0 { end + len } else { end };
    let start = if start < 0 { start + len } else { start };
    (bound(start, len), bound(end, len))
}

pub(crate) fn convert_index(len: i32, start: i32) -> usize {
    let start = if start < 0 { start + len } else { start };
    bound(start, len)
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! AST for parsed starlark files.

use std::fmt;
use std::fmt::Debug;
use std::fmt::Display;
use std::fmt::Formatter;

use allocative::Allocative;
use dupe::Dupe;

use crate::codemap::Pos;
use crate::codemap::Span;
use crate::codemap::Spanned;
use crate::syntax::lexer::TokenInt;

/// Payload types attached to AST nodes.
pub(crate) trait AstPayload: Debug {
    // TODO(nga): we don't really need `Clone` for any payload.
    type LoadPayload: Debug + Clone;
    type IdentPayload: Debug + Clone;
    type IdentAssignPayload: Debug + Clone;
    type DefPayload: Debug + Clone;
    type TypeExprPayload: Debug + Clone;
}

/// Default implementation of payload, which attaches `()` to nodes.
/// This payload is returned with AST by parser.
#[derive(Debug, Copy, Clone, Dupe, Eq, PartialEq)]
pub(crate) struct AstNoPayload;
impl AstPayload for AstNoPayload {
    type LoadPayload = ();
    type IdentPayload = ();
    type IdentAssignPayload = ();
    type DefPayload = ();
    type TypeExprPayload = ();
}

pub(crate) type Expr = ExprP<AstNoPayload>;
pub(crate) type TypeExpr = TypeExprP<AstNoPayload>;
pub(crate) type Assign = AssignP<AstNoPayload>;
pub(crate) type AssignIdent = AssignIdentP<AstNoPayload>;
pub(crate) type Ident = IdentP<AstNoPayload>;
pub(crate) type Clause = ClauseP<AstNoPayload>;
pub(crate) type ForClause = ForClauseP<AstNoPayload>;
pub(crate) type Argument = ArgumentP<AstNoPayload>;
pub(crate) type Parameter = ParameterP<AstNoPayload>;
pub(crate) type Load = LoadP<AstNoPayload>;
pub(crate) type Stmt = StmtP<AstNoPayload>;

// Boxed types used for storing information from the parsing will be used
// especially for the location of the AST item
pub(crate) type AstExprP<P> = Spanned<ExprP<P>>;
pub(crate) type AstTypeExprP<P> = Spanned<TypeExprP<P>>;
pub(crate) type AstAssignP<P> = Spanned<AssignP<P>>;
pub(crate) type AstAssignIdentP<P> = Spanned<AssignIdentP<P>>;
pub(crate) type AstIdentP<P> = Spanned<IdentP<P>>;
pub(crate) type AstArgumentP<P> = Spanned<ArgumentP<P>>;
pub(crate) type AstParameterP<P> = Spanned<ParameterP<P>>;
pub(crate) type AstStmtP<P> = Spanned<StmtP<P>>;

pub(crate) type AstExpr = AstExprP<AstNoPayload>;
pub(crate) type AstTypeExpr = AstTypeExprP<AstNoPayload>;
pub(crate) type AstAssign = AstAssignP<AstNoPayload>;
pub(crate) type AstAssignIdent = AstAssignIdentP<AstNoPayload>;
pub(crate) type AstIdent = AstIdentP<AstNoPayload>;
pub(crate) type AstArgument = AstArgumentP<AstNoPayload>;
pub(crate) type AstString = Spanned<String>;
pub(crate) type AstParameter = AstParameterP<AstNoPayload>;
pub(crate) type AstInt = Spanned<TokenInt>;
pub(crate) type AstFloat = Spanned<f64>;
pub(crate) type AstStmt = AstStmtP<AstNoPayload>;

// A trait rather than a function to allow .ast() chaining in the parser.
pub(crate) trait ToAst: Sized {
    fn ast(self, begin: usize, end: usize) -> Spanned<Self> {
        Spanned {
            span: Span::new(Pos::new(begin as u32), Pos::new(end as u32)),
            node: self,
        }
    }
}

impl<T> ToAst for T {}

#[derive(Debug, Clone)]
pub(crate) enum ArgumentP<P: AstPayload> {
    Positional(AstExprP<P>),
    Named(AstString, AstExprP<P>),
    Args(AstExprP<P>),
    KwArgs(AstExprP<P>),
}

#[derive(Debug, Clone)]
pub(crate) enum ParameterP<P: AstPayload> {
    Normal(AstAssignIdentP<P>, Option<Box<AstTypeExprP<P>>>),
    WithDefaultValue(
        AstAssignIdentP<P>,
        Option<Box<AstTypeExprP<P>>>,
        Box<AstExprP<P>>,
    ),
    NoArgs,
    Args(AstAssignIdentP<P>, Option<Box<AstTypeExprP<P>>>),
    KwArgs(AstAssignIdentP<P>, Option<Box<AstTypeExprP<P>>>),
}

impl<P: AstPayload> ParameterP<P> {
    pub(crate) fn ident(&self) -> Option<&AstAssignIdentP<P>> {
        match self {
            ParameterP::Normal(x, _)
            | ParameterP::WithDefaultValue(x, _, _)
            | ParameterP::Args(x, _)
            | ParameterP::KwArgs(x, _) => Some(x),
            ParameterP::NoArgs => None,
        }
    }
}

#[derive(Debug, Clone)]
pub(crate) enum AstLiteral {
    Int(AstInt),
    Float(AstFloat),
    String(AstString),
}

#[derive(Debug, Clone)]
pub(crate) struct LambdaP<P: AstPayload> {
    pub(crate) params: Vec<AstParameterP<P>>,
    pub(crate) body: Box<AstExprP<P>>,
    pub(crate) payload: P::DefPayload,
}

impl<P: AstPayload> LambdaP<P> {
    pub(crate) fn signature_span(&self) -> Span {
        self.params
            .iter()
            .map(|p| p.span)
            .reduce(|a, b| a.merge(b))
            .unwrap_or(
                // TODO(nga): this is not correct span.
                self.body.span,
            )
    }
}

#[derive(Debug, Clone)]
pub(crate) enum ExprP<P: AstPayload> {
    Tuple(Vec<AstExprP<P>>),
    Dot(Box<AstExprP<P>>, AstString),
    Call(Box<AstExprP<P>>, Vec<AstArgumentP<P>>),
    Index(Box<(AstExprP<P>, AstExprP<P>)>),
    Index2(Box<(AstExprP<P>, AstExprP<P>, AstExprP<P>)>),
    Slice(
        Box<AstExprP<P>>,
        Option<Box<AstExprP<P>>>,
        Option<Box<AstExprP<P>>>,
        Option<Box<AstExprP<P>>>,
    ),
    Identifier(AstIdentP<P>),
    Lambda(LambdaP<P>),
    Literal(AstLiteral),
    Not(Box<AstExprP<P>>),
    Minus(Box<AstExprP<P>>),
    Plus(Box<AstExprP<P>>),
    BitNot(Box<AstExprP<P>>),
    Op(Box<AstExprP<P>>, BinOp, Box<AstExprP<P>>),
    If(Box<(AstExprP<P>, AstExprP<P>, AstExprP<P>)>), // Order: condition, v1, v2 <=> v1 if condition else v2
    List(Vec<AstExprP<P>>),
    Dict(Vec<(AstExprP<P>, AstExprP<P>)>),
    ListComprehension(Box<AstExprP<P>>, Box<ForClauseP<P>>, Vec<ClauseP<P>>),
    DictComprehension(
        Box<(AstExprP<P>, AstExprP<P>)>,
        Box<ForClauseP<P>>,
        Vec<ClauseP<P>>,
    ),
}

/// Restricted expression at type position.
#[derive(Debug, Clone)]
pub(crate) struct TypeExprP<P: AstPayload> {
    /// Currently it is an expr.
    /// Planning to restrict it.
    /// [Context](https://fb.workplace.com/groups/buck2eng/posts/3196541547309990).
    pub(crate) expr: AstExprP<P>,
    pub(crate) payload: P::TypeExprPayload,
}

/// In some places e.g. AssignModify, the Tuple case is not allowed.
#[derive(Debug, Clone)]
pub(crate) enum AssignP<P: AstPayload> {
    // We use Tuple for both Tuple and List,
    // as these have the same semantics in Starlark.
    Tuple(Vec<AstAssignP<P>>),
    Index(Box<(AstExprP<P>, AstExprP<P>)>),
    Dot(Box<AstExprP<P>>, AstString),
    Identifier(AstAssignIdentP<P>),
}

/// Identifier in assign position.
#[derive(Debug, Eq, PartialEq, Clone)]
pub(crate) struct AssignIdentP<P: AstPayload>(pub String, pub P::IdentAssignPayload);

/// Identifier in read position, e. g. `foo` in `[foo.bar]`.
/// `foo` in `foo = 1` or `bar.foo` are **not** represented by this type.
#[derive(Debug, Eq, PartialEq, Clone)]
pub(crate) struct IdentP<P: AstPayload>(pub String, pub P::IdentPayload);

/// `load` statement.
#[derive(Debug, Clone)]
pub(crate) struct LoadP<P: AstPayload> {
    pub(crate) module: AstString,
    pub(crate) args: Vec<(AstAssignIdentP<P>, AstString)>,
    pub(crate) payload: P::LoadPayload,
}

#[derive(Debug, Clone)]
pub(crate) struct ForClauseP<P: AstPayload> {
    pub(crate) var: AstAssignP<P>,
    pub(crate) over: AstExprP<P>,
}

#[derive(Debug, Clone)]
pub(crate) enum ClauseP<P: AstPayload> {
    For(ForClauseP<P>),
    If(AstExprP<P>),
}

#[derive(Debug, Clone, Copy, Dupe, Eq, PartialEq)]
pub(crate) enum BinOp {
    Or,
    And,
    Equal,
    NotEqual,
    Less,
    Greater,
    LessOrEqual,
    GreaterOrEqual,
    In,
    NotIn,
    Subtract,
    Add,
    Multiply,
    Percent,
    Divide,
    FloorDivide,
    BitAnd,
    BitOr,
    BitXor,
    LeftShift,
    RightShift,
}

#[derive(Debug, Clone, Copy, Dupe, PartialEq, Eq)]
pub(crate) enum AssignOp {
    Add,         // +=
    Subtract,    // -=
    Multiply,    // *=
    Divide,      // /=
    FloorDivide, // //=
    Percent,     // %=
    BitAnd,      // &=
    BitOr,       // |=
    BitXor,      // ^=
    LeftShift,   // <<=
    RightShift,  // >>=
}

#[derive(Debug, Copy, Clone, Dupe, Eq, PartialEq, Allocative)]
pub enum Visibility {
    Private,
    Public,
}

#[derive(Debug)]
pub(crate) struct DefP<P: AstPayload> {
    pub(crate) name: AstAssignIdentP<P>,
    pub(crate) params: Vec<AstParameterP<P>>,
    pub(crate) return_type: Option<Box<AstTypeExprP<P>>>,
    pub(crate) body: Box<AstStmtP<P>>,
    pub(crate) payload: P::DefPayload,
}

impl<P: AstPayload> DefP<P> {
    pub(crate) fn signature_span(&self) -> Span {
        let mut span = self.name.span;
        for param in &self.params {
            span = span.merge(param.span);
        }
        if let Some(return_type) = &self.return_type {
            span = span.merge(return_type.span);
        }
        span
    }
}

#[derive(Debug)]
pub(crate) enum StmtP<P: AstPayload> {
    Break,
    Continue,
    Pass,
    Return(Option<AstExprP<P>>),
    Expression(AstExprP<P>),
    // LHS : TYPE = RHS for the fields
    Assign(AstAssignP<P>, Box<(Option<AstTypeExprP<P>>, AstExprP<P>)>),
    AssignModify(AstAssignP<P>, AssignOp, Box<AstExprP<P>>),
    Statements(Vec<AstStmtP<P>>),
    If(AstExprP<P>, Box<AstStmtP<P>>),
    IfElse(AstExprP<P>, Box<(AstStmtP<P>, AstStmtP<P>)>),
    For(AstAssignP<P>, Box<(AstExprP<P>, AstStmtP<P>)>),
    Def(DefP<P>),
    Load(LoadP<P>),
}

impl<P: AstPayload> StmtP<P> {
    /// Ensure we produce normalised Statements, rather than singleton Statements
    pub(crate) fn statements(mut xs: Vec<AstStmtP<P>>, begin: usize, end: usize) -> AstStmtP<P> {
        if xs.len() == 1 {
            xs.pop().unwrap()
        } else {
            Self::Statements(xs).ast(begin, end)
        }
    }
}

impl<P: AstPayload> ArgumentP<P> {
    pub(crate) fn expr(&self) -> &AstExprP<P> {
        match self {
            ArgumentP::Positional(x) => x,
            ArgumentP::Named(_, x) => x,
            ArgumentP::Args(x) => x,
            ArgumentP::KwArgs(x) => x,
        }
    }

    pub(crate) fn expr_mut(&mut self) -> &mut AstExprP<P> {
        match self {
            ArgumentP::Positional(x) => x,
            ArgumentP::Named(_, x) => x,
            ArgumentP::Args(x) => x,
            ArgumentP::KwArgs(x) => x,
        }
    }
}

impl Display for BinOp {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match *self {
            BinOp::Or => f.write_str(" or "),
            BinOp::And => f.write_str(" and "),
            BinOp::Equal => f.write_str(" == "),
            BinOp::NotEqual => f.write_str(" != "),
            BinOp::Less => f.write_str(" < "),
            BinOp::Greater => f.write_str(" > "),
            BinOp::LessOrEqual => f.write_str(" <= "),
            BinOp::GreaterOrEqual => f.write_str(" >= "),
            BinOp::In => f.write_str(" in "),
            BinOp::NotIn => f.write_str(" not in "),
            BinOp::Subtract => f.write_str(" - "),
            BinOp::Add => f.write_str(" + "),
            BinOp::Multiply => f.write_str(" * "),
            BinOp::Percent => f.write_str(" % "),
            BinOp::Divide => f.write_str(" / "),
            BinOp::FloorDivide => f.write_str(" // "),
            BinOp::BitAnd => f.write_str(" & "),
            BinOp::BitOr => f.write_str(" | "),
            BinOp::BitXor => f.write_str(" ^ "),
            BinOp::LeftShift => f.write_str(" << "),
            BinOp::RightShift => f.write_str(" >> "),
        }
    }
}

impl Display for AssignOp {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match *self {
            AssignOp::Add => f.write_str(" += "),
            AssignOp::Subtract => f.write_str(" -= "),
            AssignOp::Multiply => f.write_str(" *= "),
            AssignOp::Divide => f.write_str(" /= "),
            AssignOp::FloorDivide => f.write_str(" //= "),
            AssignOp::Percent => f.write_str(" %= "),
            AssignOp::BitAnd => f.write_str(" &= "),
            AssignOp::BitOr => f.write_str(" |= "),
            AssignOp::BitXor => f.write_str(" ^= "),
            AssignOp::LeftShift => f.write_str(" <<= "),
            AssignOp::RightShift => f.write_str(" >>= "),
        }
    }
}

fn comma_separated_fmt<I, F>(
    f: &mut Formatter<'_>,
    v: &[I],
    converter: F,
    for_tuple: bool,
) -> fmt::Result
where
    F: Fn(&I, &mut Formatter<'_>) -> fmt::Result,
{
    for (i, e) in v.iter().enumerate() {
        f.write_str(if i == 0 { "" } else { ", " })?;
        converter(e, f)?;
    }
    if v.len() == 1 && for_tuple {
        f.write_str(",")?;
    }
    Ok(())
}

fn fmt_string_literal(f: &mut Formatter<'_>, s: &str) -> fmt::Result {
    f.write_str("\"")?;
    for c in s.chars() {
        match c {
            '\n' => f.write_str("\\n")?,
            '\t' => f.write_str("\\t")?,
            '\r' => f.write_str("\\r")?,
            '\0' => f.write_str("\\0")?,
            '"' => f.write_str("\\\"")?,
            '\\' => f.write_str("\\\\")?,
            x => f.write_str(&x.to_string())?,
        }
    }
    f.write_str("\"")
}

impl Display for AstLiteral {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match self {
            AstLiteral::Int(i) => write!(f, "{}", &i.node),
            AstLiteral::Float(n) => write!(f, "{}", &n.node),
            AstLiteral::String(s) => fmt_string_literal(f, &s.node),
        }
    }
}

impl Display for Expr {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match self {
            Expr::Tuple(e) => {
                f.write_str("(")?;
                comma_separated_fmt(f, e, |x, f| write!(f, "{}", x.node), true)?;
                f.write_str(")")
            }
            Expr::Dot(e, s) => write!(f, "{}.{}", e.node, s.node),
            Expr::Lambda(LambdaP {
                params,
                body,
                payload: _,
            }) => {
                f.write_str("(lambda ")?;
                comma_separated_fmt(f, params, |x, f| write!(f, "{}", x.node), false)?;
                f.write_str(": ")?;
                write!(f, "{}", body.node)?;
                f.write_str(")")
            }
            Expr::Call(e, args) => {
                write!(f, "{}(", e.node)?;
                for (i, x) in args.iter().enumerate() {
                    if i != 0 {
                        f.write_str(", ")?;
                    }
                    write!(f, "{}", x.node)?;
                }
                f.write_str(")")
            }
            Expr::Index(e_i) => {
                let (e, i) = &**e_i;
                write!(f, "{}[{}]", e.node, i.node)
            }
            Expr::Index2(a_i0_i1) => {
                let (a, i0, i1) = &**a_i0_i1;
                write!(f, "{}[{}, {}]", a.node, i0.node, i1.node)
            }
            Expr::Slice(e, i1, i2, i3) => {
                write!(f, "{}[]", e.node)?;
                if let Some(x) = i1 {
                    write!(f, "{}:", x.node)?
                } else {
                    f.write_str(":")?
                }
                if let Some(x) = i2 {
                    write!(f, "{}", x.node)?
                }
                if let Some(x) = i3 {
                    write!(f, ":{}", x.node)?
                }
                Ok(())
            }
            Expr::Identifier(s) => Display::fmt(&s.node, f),
            Expr::Not(e) => write!(f, "(not {})", e.node),
            Expr::Minus(e) => write!(f, "-{}", e.node),
            Expr::Plus(e) => write!(f, "+{}", e.node),
            Expr::BitNot(e) => write!(f, "~{}", e.node),
            Expr::Op(l, op, r) => write!(f, "({}{}{})", l.node, op, r.node),
            Expr::If(cond_v1_v2) => {
                let (cond, v1, v2) = &**cond_v1_v2;
                write!(f, "({} if {} else {})", v1.node, cond.node, v2.node)
            }
            Expr::List(v) => {
                f.write_str("[")?;
                comma_separated_fmt(f, v, |x, f| write!(f, "{}", x.node), false)?;
                f.write_str("]")
            }
            Expr::Dict(v) => {
                f.write_str("{")?;
                comma_separated_fmt(f, v, |x, f| write!(f, "{}: {}", x.0.node, x.1.node), false)?;
                f.write_str("}")
            }
            Expr::ListComprehension(e, for_, c) => {
                write!(f, "[{}", e.node)?;
                write!(f, "{}", for_)?;
                for x in c {
                    write!(f, "{}", x)?;
                }
                f.write_str("]")
            }
            Expr::DictComprehension(k_v, for_, c) => {
                let (k, v) = &**k_v;
                write!(f, "{{{}: {}", k.node, v.node)?;
                write!(f, "{}", for_)?;
                for x in c {
                    write!(f, "{}", x)?;
                }
                f.write_str("}}")
            }
            Expr::Literal(x) => write!(f, "{}", x),
        }
    }
}

impl Display for TypeExpr {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        Display::fmt(&self.expr.node, f)
    }
}

impl Display for Assign {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match self {
            Assign::Tuple(e) => {
                f.write_str("(")?;
                comma_separated_fmt(f, e, |x, f| write!(f, "{}", x.node), true)?;
                f.write_str(")")
            }
            Assign::Dot(e, s) => write!(f, "{}.{}", e.node, s.node),
            Assign::Index(e_i) => {
                let (e, i) = &**e_i;
                write!(f, "{}[{}]", e.node, i.node)
            }
            Assign::Identifier(s) => write!(f, "{}", s.node),
        }
    }
}

impl Display for AssignIdent {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl Display for Ident {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl Display for Argument {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match self {
            Argument::Positional(s) => write!(f, "{}", s.node),
            Argument::Named(s, e) => write!(f, "{} = {}", s.node, e.node),
            Argument::Args(s) => write!(f, "*{}", s.node),
            Argument::KwArgs(s) => write!(f, "**{}", s.node),
        }
    }
}

impl Display for Parameter {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        let (prefix, name, typ, default) = match self {
            Parameter::Normal(s, t) => ("", s, t, None),
            Parameter::WithDefaultValue(s, t, e) => ("", s, t, Some(e)),
            Parameter::NoArgs => return write!(f, "*"),
            Parameter::Args(s, t) => ("*", s, t, None),
            Parameter::KwArgs(s, t) => ("**", s, t, None),
        };
        write!(f, "{}{}", prefix, name.node)?;
        if let Some(t) = typ {
            write!(f, ": {}", t.node)?;
        }
        if let Some(d) = default {
            write!(f, " = {}", d.node)?;
        }
        Ok(())
    }
}

impl Display for ForClause {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        write!(f, " for {} in {}", self.var.node, self.over.node)
    }
}

impl Display for Clause {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match self {
            Clause::For(x) => write!(f, "{}", x),
            Clause::If(x) => write!(f, " if {}", x.node),
        }
    }
}

impl Stmt {
    fn fmt_with_tab(&self, f: &mut Formatter<'_>, tab: String) -> fmt::Result {
        match self {
            Stmt::Break => writeln!(f, "{}break", tab),
            Stmt::Continue => writeln!(f, "{}continue", tab),
            Stmt::Pass => writeln!(f, "{}pass", tab),
            Stmt::Return(Some(e)) => writeln!(f, "{}return {}", tab, e.node),
            Stmt::Return(None) => writeln!(f, "{}return", tab),
            Stmt::Expression(e) => writeln!(f, "{}{}", tab, e.node),
            Stmt::Assign(l, ty_r) => {
                let (ty, r) = &**ty_r;
                write!(f, "{}{} ", tab, l.node)?;
                if let Some(ty) = ty {
                    write!(f, ": {} ", ty.node)?;
                }
                writeln!(f, "= {}", r.node)
            }
            Stmt::AssignModify(l, op, r) => writeln!(f, "{}{}{}{}", tab, l.node, op, r.node),
            Stmt::Statements(v) => {
                for s in v {
                    s.node.fmt_with_tab(f, tab.clone())?;
                }
                Ok(())
            }
            Stmt::If(cond, suite) => {
                writeln!(f, "{}if {}:", tab, cond.node)?;
                suite.node.fmt_with_tab(f, tab + "  ")
            }
            Stmt::IfElse(cond, suite_1_2) => {
                let (suite1, suite2) = &**suite_1_2;
                writeln!(f, "{}if {}:", tab, cond.node)?;
                suite1.node.fmt_with_tab(f, tab.clone() + "  ")?;
                writeln!(f, "{}else:", tab)?;
                suite2.node.fmt_with_tab(f, tab + "  ")
            }
            Stmt::For(bind, coll_suite) => {
                let (coll, suite) = &**coll_suite;
                writeln!(f, "{}for {} in {}:", tab, bind.node, coll.node)?;
                suite.node.fmt_with_tab(f, tab + "  ")
            }
            Stmt::Def(DefP {
                name,
                params,
                return_type,
                body,
                payload: _,
            }) => {
                write!(f, "{}def {}(", tab, name.node)?;
                comma_separated_fmt(f, params, |x, f| write!(f, "{}", x.node), false)?;
                f.write_str(")")?;
                if let Some(rt) = return_type {
                    write!(f, " -> {}", rt.node)?;
                }
                f.write_str(":\n")?;
                body.node.fmt_with_tab(f, tab + "  ")
            }
            Stmt::Load(load) => {
                write!(f, "{}load(", tab)?;
                fmt_string_literal(f, &load.module.node)?;
                comma_separated_fmt(
                    f,
                    &load.args,
                    |x, f| {
                        write!(f, "{} = ", x.0.node)?;
                        fmt_string_literal(f, &(x.1.node))
                    },
                    false,
                )?;
                f.write_str(")\n")
            }
        }
    }
}

impl Display for Stmt {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        self.fmt_with_tab(f, "".to_owned())
    }
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use std::slice;
use std::str::Chars;

pub(crate) struct CursorBytes<'a>(&'a str, slice::Iter<'a, u8>);

impl<'a> CursorBytes<'a> {
    pub fn new(x: &'a str) -> Self {
        Self(x, x.as_bytes().iter())
    }

    pub fn next(&mut self) -> Option<u8> {
        self.1.next().copied()
    }

    // If it returns a value great than 127, it should not be trusted
    pub fn next_char(&mut self) -> Option<char> {
        self.next().map(|x| x as char)
    }

    pub fn pos(&self) -> usize {
        self.0.len() - self.1.as_slice().len()
    }
}

pub(crate) struct CursorChars<'a>(&'a str, Chars<'a>);

impl<'a> CursorChars<'a> {
    pub fn new_offset(x: &'a str, offset: usize) -> Self {
        Self(x, x[offset..].chars())
    }

    pub fn next(&mut self) -> Option<char> {
        self.1.next()
    }

    /// Call `unnext` to put back a character you grabbed with next.
    /// It is an error if the character isn't what you declared.
    pub fn unnext(&mut self, c: char) {
        let pos = self.pos();
        self.1 = self.0[pos - c.len_utf8()..].chars();
        debug_assert_eq!(self.peek(), Some(c))
    }

    pub fn peek(&self) -> Option<char> {
        self.1.as_str().chars().next()
    }

    pub fn pos(&self) -> usize {
        self.0.len() - self.1.as_str().len()
    }
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use dupe::Dupe;
use thiserror::Error;

use crate::codemap::CodeMap;
use crate::codemap::Pos;
use crate::codemap::Span;
use crate::codemap::Spanned;
use crate::eval::compiler::EvalException;
use crate::syntax::ast::Expr;
use crate::syntax::ast::TypeExpr;
use crate::syntax::ast::TypeExprP;
use crate::syntax::ast::Visibility;
use crate::syntax::type_expr::TypeExprUnpackP;

#[derive(Error, Debug)]
pub(crate) enum DialectError {
    #[error("`def` is not allowed in this dialect")]
    Def,
    #[error("`lambda` is not allowed in this dialect")]
    Lambda,
    #[error("`load` is not allowed in this dialect")]
    Load,
    #[error("* keyword-only-arguments is not allowed in this dialect")]
    KeywordOnlyArguments,
    #[error("type annotations are not allowed in this dialect")]
    Types,
}

/// How to handle type annotations in Starlark.
#[derive(Debug, Clone, Copy, Dupe, Eq, PartialEq, Hash)]
pub enum DialectTypes {
    /// Prohibit types at parse time.
    Disable,
    /// Allow types at parse time, but ignore types at runtime.
    ParseOnly,
    /// Check types at runtime.
    Enable,
}

/// Starlark language features to enable, e.g. [`Standard`](Dialect::Standard) to follow the Starlark standard.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct Dialect {
    /// Are `def` statements permitted.
    /// Enabled in both [`Standard`](Dialect::Standard) and [`Extended`](Dialect::Extended).
    pub enable_def: bool,
    /// Are `lambda` expressions permitted.
    /// Enabled in both [`Standard`](Dialect::Standard) and [`Extended`](Dialect::Extended).
    pub enable_lambda: bool,
    /// Are `load` statements permitted.
    /// Enabled in both [`Standard`](Dialect::Standard) and [`Extended`](Dialect::Extended).
    pub enable_load: bool,
    /// Are `*` keyword-only arguments allowed as per [PEP 3102](https://www.python.org/dev/peps/pep-3102/).
    /// Only enabled in [`Extended`](Dialect::Extended).
    pub enable_keyword_only_arguments: bool,
    /// Are expressions allowed in type positions as per [PEP 484](https://www.python.org/dev/peps/pep-0484/).
    /// Only enabled in [`Extended`](Dialect::Extended).
    pub enable_types: DialectTypes,
    /// Do `load()` statements reexport their definition.
    /// Enabled in both [`Standard`](Dialect::Standard) and [`Extended`](Dialect::Extended),
    /// but may change in future definitions of the standard.
    pub enable_load_reexport: bool,
    /// Are `for`, `if` and other statements allowed at the top level.
    /// Only enabled in [`Extended`](Dialect::Extended).
    pub enable_top_level_stmt: bool,
    /// Like `#[non_exhaustive]`, but allows struct expression.
    ///
    /// [Explanation](https://github.com/rust-lang/rust-clippy/issues/6559).
    pub _non_exhaustive: (),
}

impl Default for Dialect {
    fn default() -> Dialect {
        Dialect::Standard
    }
}

// These are morally enumerations, so give them enumeration-like names
// even though they are actually global constants
#[allow(non_upper_case_globals)]
impl Dialect {
    /// Follow the [Starlark language standard](https://github.com/bazelbuild/starlark/blob/master/spec.md) as much as possible.
    ///
    /// This is also returned by [`Dialect::default()`](Dialect::default).
    pub const Standard: Self = Self {
        enable_def: true,
        enable_lambda: true,
        enable_load: true,
        enable_keyword_only_arguments: false,
        enable_types: DialectTypes::Disable,
        enable_load_reexport: true, // But they plan to change it
        enable_top_level_stmt: false,
        _non_exhaustive: (),
    };

    /// A superset of [`Standard`](Dialect::Standard), including extra features (types, top-level statements etc).
    pub const Extended: Self = Self {
        enable_def: true,
        enable_lambda: true,
        enable_load: true,
        enable_keyword_only_arguments: true,
        enable_types: DialectTypes::Enable,
        enable_load_reexport: true,
        enable_top_level_stmt: true,
        _non_exhaustive: (),
    };
}

fn err<T>(codemap: &CodeMap, span: Span, err: DialectError) -> Result<T, EvalException> {
    Err(EvalException::new(err.into(), span, codemap))
}

impl Dialect {
    pub(crate) fn check_lambda<T>(
        &self,
        codemap: &CodeMap,
        x: Spanned<T>,
    ) -> Result<Spanned<T>, EvalException> {
        if self.enable_lambda {
            Ok(x)
        } else {
            err(codemap, x.span, DialectError::Lambda)
        }
    }

    pub(crate) fn check_def<T>(
        &self,
        codemap: &CodeMap,
        x: Spanned<T>,
    ) -> Result<Spanned<T>, EvalException> {
        if self.enable_def {
            Ok(x)
        } else {
            err(codemap, x.span, DialectError::Def)
        }
    }

    pub(crate) fn check_keyword_only_arguments<T>(
        &self,
        codemap: &CodeMap,
        begin: usize,
        end: usize,
        x: T,
    ) -> Result<T, EvalException> {
        let span = Span::new(Pos::new(begin as u32), Pos::new(end as u32));
        if self.enable_keyword_only_arguments {
            Ok(x)
        } else {
            err(codemap, span, DialectError::KeywordOnlyArguments)
        }
    }

    pub(crate) fn check_type(
        &self,
        codemap: &CodeMap,
        x: Spanned<Expr>,
    ) -> Result<Spanned<TypeExpr>, EvalException> {
        let span = x.span;
        if self.enable_types == DialectTypes::Disable {
            return err(codemap, x.span, DialectError::Types);
        }

        TypeExprUnpackP::unpack(&x, codemap)?;

        Ok(x.into_map(|node| TypeExprP {
            expr: Spanned { span, node },
            payload: (),
        }))
    }

    pub(crate) fn load_visibility(&self) -> Visibility {
        if self.enable_load_reexport {
            Visibility::Public
        } else {
            Visibility::Private
        }
    }
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use crate::assert;
use crate::assert::Assert;
use crate::slice_vec_ext::SliceExt;
use crate::syntax::ast::Stmt;

#[test]
fn test_empty() {
    assert_eq!(assert::parse("\n"), "");
    assert_eq!(assert::parse(""), "");
    assert_eq!(assert::parse("    "), "");
    assert_eq!(assert::parse("    \n"), "");
    // Empty trailing bits
    assert_eq!(assert::parse("x = 1\n   "), "x = 1\n");
    assert_eq!(assert::parse("x = 1\n   \n"), "x = 1\n");
}

#[test]
fn test_assign_op() {
    assert_eq!(assert::parse("x = 1\n"), "x = 1\n");
    assert_eq!(assert::parse("x += 1\n"), "x += 1\n");
    assert_eq!(assert::parse("x -= 1\n"), "x -= 1\n");
    assert_eq!(assert::parse("x *= 1\n"), "x *= 1\n");
    assert_eq!(assert::parse("x /= 1\n"), "x /= 1\n");
    assert_eq!(assert::parse("x //= 1\n"), "x //= 1\n");
    assert_eq!(assert::parse("x %= 1\n"), "x %= 1\n");
    assert_eq!(assert::parse("x &= 1\n"), "x &= 1\n");
    assert_eq!(assert::parse("x |= 1\n"), "x |= 1\n");
    assert_eq!(assert::parse("x ^= 1\n"), "x ^= 1\n");
    assert_eq!(assert::parse("x <<= 1\n"), "x <<= 1\n");
    assert_eq!(assert::parse("x >>= 1\n"), "x >>= 1\n");
}

#[test]
fn test_top_level_comment() {
    assert_eq!(assert::parse("# Test"), "");
}

#[test]
fn test_top_level_load() {
    let want = "load(\"//top/level/load.bzl\"top-level = \"top-level\")\n";
    assert_eq!(
        assert::parse("\nload(\"//top/level/load.bzl\", \"top-level\")\n"),
        want
    );
    assert_eq!(
        assert::parse("\nload(\"//top/level/load.bzl\", \"top-level\")"),
        want
    );
    assert_eq!(
        assert::parse("\nload(\n  \"//top/level/load.bzl\",\n  \"top-level\",\n)\n"),
        want
    );
}

#[test]
fn test_top_level_assignation() {
    assert_eq!(
        assert::parse("\n_ASSIGNATION = 'top-level'\n"),
        "_ASSIGNATION = \"top-level\"\n"
    );
}

#[test]
fn test_top_level_docstring() {
    assert_eq!(
        assert::parse("\n\"\"\"Top-level docstring\"\"\"\n"),
        "\"Top-level docstring\"\n"
    );
}

#[test]
fn test_top_level_def() {
    assert_eq!(
        assert::parse("def toto():\n  pass\n"),
        "def toto():\n  pass\n"
    );
    let mut a = Assert::new();
    a.dialect_set(|x| x.enable_def = false);
    a.parse_fail("!def toto():\n  pass\n!");
    // no new line at end of file
    assert_eq!(
        assert::parse("def toto():\n  pass"),
        "def toto():\n  pass\n"
    );
    assert_eq!(
        assert::parse("def toto():\n  pass\ndef titi(): return 1"),
        "def toto():\n  pass\ndef titi():\n  return 1\n"
    );
    assert_eq!(
        assert::parse("def toto():\n  pass\n\ndef titi(): return 1"),
        "def toto():\n  pass\ndef titi():\n  return 1\n"
    );
    assert_eq!(assert::parse("def t():\n\n  pass"), "def t():\n  pass\n");
}

#[test]
fn test_top_level_statements() {
    let mut a = Assert::new();
    a.dialect_set(|x| x.enable_top_level_stmt = false);
    a.parse_fail("x = 1\n!if x == 1:\n  x = 2\n!x = 3");
    a.parse_fail("x = 1\n!for x in []:\n   pass\n!");
    assert_eq!(a.parse("pass"), "pass\n");

    assert_eq!(
        assert::parse("if x == 1:\n  x = 2"),
        "if (x == 1):\n  x = 2\n"
    );
    assert_eq!(
        assert::parse("for x in []:\n   pass\n"),
        "for x in []:\n  pass\n"
    );
    assert_eq!(assert::parse("pass"), "pass\n");
}

#[test]
fn test_top_level_def_with_docstring() {
    assert_eq!(
        assert::parse(
            "\"\"\"Top-level docstring\"\"\"

def toto():
  pass
"
        ),
        "\"Top-level docstring\"\ndef toto():\n  pass\n"
    );
}

#[test]
fn test_ifelse() {
    assert_eq!(
        assert::parse("def d():\n  if True:\n    a\n  else:\n    b"),
        "def d():\n  if True:\n    a\n  else:\n    b\n"
    );
}

#[test]
fn test_kwargs_passing() {
    assert_eq!(
        assert::parse("f(x, *a, **b); f(x, *a, **{a:b}); f(x, *[a], **b)"),
        "f(x, *a, **b)\nf(x, *a, **{a: b})\nf(x, *[a], **b)\n"
    );
}

#[test]
fn test_unary_op() {
    assert_eq!(assert::parse("a = -1"), "a = -1\n");
    assert_eq!(assert::parse("a = +1"), "a = +1\n");
    assert_eq!(assert::parse("a = -a"), "a = -a\n");
    assert_eq!(assert::parse("a = +a"), "a = +a\n");
}

#[test]
fn test_tuples() {
    assert_eq!(assert::parse("a = (-1)"), "a = -1\n"); // Not a tuple
    assert_eq!(assert::parse("a = (+1,)"), "a = (+1,)\n"); // But this is one
    assert_eq!(assert::parse("a = ()"), "a = ()\n");
}

#[test]
fn test_return() {
    assert_eq!(
        assert::parse("def fn(): return 1"),
        "def fn():\n  return 1\n"
    );
    assert_eq!(
        assert::parse("def fn(): return a()"),
        "def fn():\n  return a()\n"
    );
    assert_eq!(assert::parse("def fn(): return"), "def fn():\n  return\n");
}

// Regression test for https://github.com/google/starlark-rust/issues/44.
#[test]
fn test_optional_whitespace() {
    assert_eq!(assert::parse("6 or()"), "(6 or ())\n");
    assert_eq!(assert::parse("6or()"), "(6 or ())\n");
}

// Regression test for https://github.com/google/starlark-rust/issues/56.
#[test]
fn test_optional_whitespace_after_0() {
    assert_eq!(assert::parse("0in[1,2,3]"), "(0 in [1, 2, 3])\n");
}

#[test]
fn test_blank_line() {
    assert_eq!(
        assert::parse(
            r#"
def foo():
  bar

  bar
"#
        ),
        "def foo():\n  bar\n  bar\n"
    )
}

#[test]
fn test_fncall_span() {
    let content = r#"def fn(a):
  fail(a)

fn(1)

fail(2)
"#;
    let ast = assert::parse_ast(content);
    match &ast.statement.node {
        Stmt::Statements(xs) => {
            let lines = xs.map(|x| ast.codemap.resolve_span(x.span).begin_line);
            assert_eq!(lines, vec![0, 3, 5])
        }
        _ => panic!("Expected to parse as statements"),
    }
}

#[test]
fn test_comprehension() {
    assert_eq!(
        assert::parse("[x for x in range(12) if x % 2 == 0 if x % 3 == 0]"),
        "[x for x in range(12) if ((x % 2) == 0) if ((x % 3) == 0)]\n"
    );
    assert_eq!(
        assert::parse("[x + 7 for x in range(12) for y in range(10)]"),
        "[(x + 7) for x in range(12) for y in range(10)]\n"
    );
}

#[test]
fn test_lambda() {
    assert_eq!(
        assert::parse("x = lambda y: y + 1"),
        "x = (lambda y: (y + 1))\n"
    );
    let mut a = Assert::new();
    a.dialect_set(|x| x.enable_lambda = false);
    a.parse_fail("x = !lambda y: y + 1!");
    assert_eq!(
        assert::parse("(lambda y: x == 1)(1)"),
        "(lambda y: (x == 1))(1)\n"
    );
    assert_eq!(
        assert::parse("(lambda x: x or 1)(1)"),
        "(lambda x: (x or 1))(1)\n"
    );
    assert_eq!(
        assert::parse("f = lambda x, y: x * y"),
        "f = (lambda x, y: (x * y))\n"
    );
    assert_eq!(assert::parse("lambda x: True"), "(lambda x: True)\n");
    assert_eq!(assert::parse("lambda: True"), "(lambda : True)\n");
    assert_eq!(
        assert::parse("f(lambda x, y=1, *args, **kwargs: x + y + z)"),
        "f((lambda x, y = 1, *args, **kwargs: ((x + y) + z)))\n"
    );
    assert_eq!(
        assert::parse("[x for x in [1, 2] if (lambda : 3 if True else 4)]"),
        "[x for x in [1, 2] if (lambda : (3 if True else 4))]\n"
    );
    // Note that Python3 and Go Starlark can both parse the line below,
    // but we can't, and the official Python3 grammar doesn't appear to be
    // able to either (we follow it in this position). Fortunately, its
    // a mostly meaningless program.
    // [x for x in [1, 2] if lambda : None]
}

#[test]
fn test_nested_def() {
    assert_eq!(
        assert::parse("def foo(x):\n  def bar(y): return y\n  return bar(x)"),
        "def foo(x):\n  def bar(y):\n    return y\n  return bar(x)\n"
    );
}

#[test]
fn test_op_associativity() {
    // Normal operators are left associative
    assert_eq!(
        assert::parse("1 or 2 or 3 or 4"),
        "(((1 or 2) or 3) or 4)\n"
    );
    assert_eq!(
        assert::parse("1 and 2 and 3 and 4"),
        "(((1 and 2) and 3) and 4)\n"
    );
    assert_eq!(assert::parse("1 | 2 | 3"), "((1 | 2) | 3)\n");
    assert_eq!(assert::parse("1 + 2 + 3"), "((1 + 2) + 3)\n");
    assert_eq!(assert::parse("1 * 2 * 3"), "((1 * 2) * 3)\n");
    // Comparisons are not associative
    // TODO - create a better error message for this case
    assert::fail("0 <= 1 < 2", "Parse error");
}

#[test]
fn test_bad_assignment() {
    assert::parse_fail("[!x or y!] = 1");
    assert::parse_fail("![x]! += 1");
}

#[test]
fn test_test_list_in_index_expr() {
    assert_eq!(assert::parse("x[1, 2]"), "x[1, 2]\n");
    assert::parse_fail("!x[1, 2]! = 3");
}


/*
 * Copyright 2018 The Starlark in Rust Authors.
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

use std::char;
use std::collections::VecDeque;
use std::fmt;
use std::fmt::Display;

use logos::Logos;
use thiserror::Error;

use crate::codemap::CodeMap;
use crate::codemap::Pos;
use crate::codemap::Span;
use crate::eval::compiler::EvalException;
use crate::syntax::cursors::CursorBytes;
use crate::syntax::cursors::CursorChars;
use crate::syntax::dialect::Dialect;
use crate::values::types::int_or_big::StarlarkInt;

#[derive(Error, Debug)]
pub(crate) enum LexemeError {
    #[error("Parse error: incorrect indentation")]
    Indentation,
    #[error("Parse error: invalid input `{0}`")]
    InvalidInput(String),
    #[error("Parse error: tabs are not allowed")]
    InvalidTab,
    #[error("Parse error: unfinished string literal")]
    UnfinishedStringLiteral,
    #[error("Parse error: invalid string escape sequence `{0}`")]
    InvalidEscapeSequence(String),
    #[error("Parse error: missing string escape sequence, only saw `\\`")]
    EmptyEscapeSequence,
    #[error("Parse error: cannot use reserved keyword `{0}`")]
    ReservedKeyword(String),
    #[error("Parse error: integer cannot have leading 0, got `{0}`")]
    StartsZero(String),
    #[error("Parse error: failed to parse integer: `{0}`")]
    IntParse(String),
}

type Lexeme = Result<(usize, Token, usize), EvalException>;

pub(crate) struct Lexer<'a> {
    // Information for spans
    codemap: CodeMap,
    // Other info
    indent_levels: Vec<usize>,
    /// Lexemes that have been generated but not yet returned
    buffer: VecDeque<Lexeme>,
    parens: isize, // Number of parens we have seen
    lexer: logos::Lexer<'a, Token>,
    done: bool,
}

impl<'a> Lexer<'a> {
    pub fn new(input: &'a str, _dialect: &Dialect, codemap: CodeMap) -> Self {
        let lexer = Token::lexer(input);
        let mut lexer2 = Self {
            codemap,
            // Aim to size all the buffers such that they never resize
            indent_levels: Vec::with_capacity(20),
            buffer: VecDeque::with_capacity(10),
            lexer,
            parens: 0,
            done: false,
        };
        if let Err(e) = lexer2.calculate_indent() {
            lexer2.buffer.push_back(Err(e));
        }
        lexer2
    }

    fn err_pos<T>(&self, msg: LexemeError, pos: usize) -> Result<T, EvalException> {
        self.err_span(msg, pos, pos)
    }

    fn err_span<T>(&self, msg: LexemeError, start: usize, end: usize) -> Result<T, EvalException> {
        Err(EvalException::new(
            msg.into(),
            Span::new(Pos::new(start as u32), Pos::new(end as u32)),
            &self.codemap,
        ))
    }

    fn err_now<T>(&self, msg: fn(String) -> LexemeError) -> Result<T, EvalException> {
        self.err_span(
            msg(self.lexer.slice().to_owned()),
            self.lexer.span().start,
            self.lexer.span().end,
        )
    }

    /// We have just seen a newline, read how many indents we have
    /// and then set self.indent properly
    fn calculate_indent(&mut self) -> Result<(), EvalException> {
        // consume tabs and spaces, output the indentation levels
        let mut it = CursorBytes::new(self.lexer.remainder());
        let mut spaces = 0;
        let mut tabs = 0;
        let mut indent_start = self.lexer.span().end;
        loop {
            match it.next_char() {
                None => {
                    self.lexer.bump(it.pos());
                    return Ok(());
                }
                Some(' ') => {
                    spaces += 1;
                }
                Some('\t') => {
                    tabs += 1;
                }
                Some('\n') => {
                    // A line that is entirely blank gets emitted as a newline, and then
                    // we don't consume the subsequent newline character.
                    self.lexer.bump(it.pos() - 1);
                    return Ok(());
                }
                Some('\r') => {
                    // We just ignore these entirely
                }
                Some('#') => {
                    // A line that is all comments doesn't get emitted at all
                    // Skip until the next newline
                    // Remove skip now, so we can freely add it on later
                    spaces = 0;
                    tabs = 0;
                    loop {
                        match it.next_char() {
                            None => {
                                self.lexer.bump(it.pos());
                                return Ok(());
                            }
                            Some('\n') => break, // only the inner loop
                            Some(_) => {}
                        }
                    }
                    indent_start = self.lexer.span().end + it.pos();
                }
                _ => break,
            }
        }
        self.lexer.bump(it.pos() - 1); // last character broke us out the loop
        let indent = spaces + tabs * 8;
        if tabs > 0 {
            return self.err_pos(LexemeError::InvalidTab, self.lexer.span().start);
        }
        let now = self.indent_levels.last().copied().unwrap_or(0);

        if indent > now {
            self.indent_levels.push(indent);
            let span = self.lexer.span();
            self.buffer
                .push_back(Ok((indent_start, Token::Indent, span.end)));
        } else if indent < now {
            let mut dedents = 1;
            self.indent_levels.pop().unwrap();
            loop {
                let now = self.indent_levels.last().copied().unwrap_or(0);
                if now == indent {
                    break;
                } else if now > indent {
                    dedents += 1;
                    self.indent_levels.pop().unwrap();
                } else {
                    let pos = self.lexer.span();
                    return self.err_span(LexemeError::Indentation, pos.start, pos.end);
                }
            }
            for _ in 0..dedents {
                // We must declare each dedent is only a position, so multiple adjacent dedents don't overlap
                self.buffer
                    .push_back(Ok((indent_start, Token::Dedent, indent_start)))
            }
        }
        Ok(())
    }

    fn wrap(&mut self, token: Token) -> Option<Lexeme> {
        let span = self.lexer.span();
        Some(Ok((span.start, token, span.end)))
    }

    // We've potentially seen one character, now consume between min and max elements of iterator
    // and treat it as an int in base radix
    fn escape_char(it: &mut CursorChars, min: usize, max: usize, radix: u32) -> Result<char, ()> {
        let mut value = 0u32;
        let mut count = 0;
        while count < max {
            match it.next() {
                None => {
                    if count >= min {
                        break;
                    } else {
                        return Err(());
                    }
                }
                Some(c) => match c.to_digit(radix) {
                    None => {
                        if count >= min {
                            it.unnext(c);
                            break;
                        } else {
                            return Err(());
                        }
                    }
                    Some(v) => {
                        count += 1;
                        value = (value * radix) + v;
                    }
                },
            }
        }
        char::from_u32(value).ok_or(())
    }

    // We have seen a '\' character, now parse what comes next
    fn escape(it: &mut CursorChars, res: &mut String) -> Result<(), ()> {
        match it.next() {
            Some('n') => res.push('\n'),
            Some('r') => res.push('\r'),
            Some('t') => res.push('\t'),
            Some('a') => res.push('\x07'),
            Some('b') => res.push('\x08'),
            Some('f') => res.push('\x0C'),
            Some('v') => res.push('\x0B'),
            Some('\n') => {}
            Some('\r') => {
                // Windows newline incoming, we expect a \n next, which we can ignore
                if it.next() != Some('\n') {
                    // A random \r character happened, let's declare an error, but we're just confused here
                    return Err(());
                }
            }
            Some('x') => res.push(Self::escape_char(it, 2, 2, 16)?),
            Some('u') => res.push(Self::escape_char(it, 4, 4, 16)?),
            Some('U') => res.push(Self::escape_char(it, 8, 8, 16)?),
            Some(c) => match c {
                '0'..='7' => {
                    it.unnext(c);
                    res.push(Self::escape_char(it, 1, 3, 8)?)
                }
                '"' | '\'' | '\\' => res.push(c),
                _ => {
                    res.push('\\');
                    res.push(c);
                }
            },
            None => {
                return Err(());
            }
        };
        Ok(())
    }

    // String parsing is a hot-spot, so parameterise by a `stop` function which gets
    // specialised for each variant
    fn string(&mut self, triple: bool, raw: bool, mut stop: impl FnMut(char) -> bool) -> Lexeme {
        // We have seen an opening quote, which is either ' or "
        // If triple is true, it was a triple quote
        // stop lets us know when a string ends.

        // Before the first quote character
        let string_start = self.lexer.span().start;
        // After the first quote character, but before any contents or it tracked stuff
        let mut string_end = self.lexer.span().end;

        let mut it = CursorBytes::new(self.lexer.remainder());
        let it2;

        if triple {
            it.next();
            it.next();
        }
        let contents_start = it.pos();

        // Take the fast path as long as the result is a slice of the original, with no changes.
        let mut res;
        loop {
            match it.next_char() {
                None => {
                    return self.err_span(
                        LexemeError::UnfinishedStringLiteral,
                        string_start,
                        string_end + it.pos(),
                    );
                }
                Some(c) => {
                    if stop(c) {
                        let contents_end = it.pos() - if triple { 3 } else { 1 };
                        let contents = &self.lexer.remainder()[contents_start..contents_end];
                        self.lexer.bump(it.pos());
                        return Ok((
                            string_start,
                            Token::String(contents.to_owned()),
                            string_end + it.pos(),
                        ));
                    } else if c == '\\' || c == '\r' || (c == '\n' && !triple) {
                        res = String::with_capacity(it.pos() + 10);
                        res.push_str(&self.lexer.remainder()[contents_start..it.pos() - 1]);
                        it2 = CursorChars::new_offset(self.lexer.remainder(), it.pos() - 1);
                        break;
                    }
                }
            }
        }

        // We bailed out of the fast path, that means we now accumulate character by character,
        // might have an error or be dealing with escape characters.
        let mut it = it2;
        while let Some(c) = it.next() {
            if stop(c) {
                self.lexer.bump(it.pos());
                if triple {
                    res.truncate(res.len() - 2);
                }
                return Ok((string_start, Token::String(res), string_end + it.pos()));
            }
            match c {
                '\n' if !triple => {
                    // Will raise an error about out of chars.
                    // But don't include the final \n in the count.
                    string_end -= 1;
                    break;
                }
                '\r' => {
                    // We just ignore these in all modes
                }
                '\\' => {
                    if raw {
                        match it.next() {
                            Some(c) => {
                                if c != '\'' && c != '"' {
                                    res.push('\\');
                                }
                                res.push(c);
                            }
                            _ => break, // Out of chars
                        }
                    } else {
                        let pos = it.pos();
                        if Self::escape(&mut it, &mut res).is_err() {
                            let bad = self.lexer.remainder()[pos..it.pos()].to_owned();
                            return self.err_span(
                                if bad.is_empty() {
                                    LexemeError::EmptyEscapeSequence
                                } else {
                                    LexemeError::InvalidEscapeSequence(bad)
                                },
                                string_end + pos - 1,
                                string_end + it.pos(),
                            );
                        }
                    }
                }
                c => res.push(c),
            }
        }

        // We ran out of characters
        self.err_span(
            LexemeError::UnfinishedStringLiteral,
            string_start,
            string_end + it.pos(),
        )
    }

    fn int(&self, s: &str, radix: u32) -> Lexeme {
        let span = self.lexer.span();
        match StarlarkInt::from_str_radix(s, radix) {
            Ok(i) => Ok((span.start, Token::Int(TokenInt(i)), span.end)),
            Err(_) => self.err_now(LexemeError::IntParse),
        }
    }

    pub fn next(&mut self) -> Option<Lexeme> {
        loop {
            // Note that this function doesn't always return - a few branches use `continue`
            // to always go round the loop again.
            return if let Some(x) = self.buffer.pop_front() {
                Some(x)
            } else if self.done {
                None
            } else {
                match self.lexer.next() {
                    None => {
                        self.done = true;
                        let pos = self.lexer.span().end;
                        for _ in 0..self.indent_levels.len() {
                            self.buffer.push_back(Ok((pos, Token::Dedent, pos)))
                        }
                        self.indent_levels.clear();
                        self.wrap(Token::Newline)
                    }
                    Some(token) => match token {
                        Token::Tabs => {
                            self.buffer.push_back(
                                self.err_pos(LexemeError::InvalidTab, self.lexer.span().start),
                            );
                            continue;
                        }
                        Token::Newline => {
                            if self.parens == 0 {
                                let span = self.lexer.span();
                                if let Err(e) = self.calculate_indent() {
                                    return Some(Err(e));
                                }
                                Some(Ok((span.start, Token::Newline, span.end)))
                            } else {
                                continue;
                            }
                        }
                        Token::Reserved => Some(self.err_now(LexemeError::ReservedKeyword)),
                        Token::Error => Some(self.err_now(LexemeError::InvalidInput)),
                        Token::RawDecInt => {
                            let s = self.lexer.slice();
                            if s.len() > 1 && &s[0..1] == "0" {
                                return Some(self.err_now(LexemeError::StartsZero));
                            }
                            Some(self.int(s, 10))
                        }
                        Token::RawOctInt => {
                            let s = self.lexer.slice();
                            assert!(s.starts_with("0o") || s.starts_with("0O"));
                            Some(self.int(&s[2..], 8))
                        }
                        Token::RawHexInt => {
                            let s = self.lexer.slice();
                            assert!(s.starts_with("0x") || s.starts_with("0X"));
                            Some(self.int(&s[2..], 16))
                        }
                        Token::RawBinInt => {
                            let s = self.lexer.slice();
                            assert!(s.starts_with("0b") || s.starts_with("0B"));
                            Some(self.int(&s[2..], 2))
                        }
                        Token::Int(..) => unreachable!("Lexer does not produce Int tokens"),
                        Token::RawDoubleQuote => {
                            let raw = self.lexer.span().len() == 2;
                            if self.lexer.remainder().starts_with("\"\"") {
                                let mut qs = 0;
                                Some(self.string(true, raw, |c| {
                                    if c == '\"' {
                                        qs += 1;
                                        qs == 3
                                    } else {
                                        qs = 0;
                                        false
                                    }
                                }))
                            } else {
                                Some(self.string(false, raw, |c| c == '\"'))
                            }
                        }
                        Token::RawSingleQuote => {
                            let raw = self.lexer.span().len() == 2;
                            if self.lexer.remainder().starts_with("''") {
                                let mut qs = 0;
                                Some(self.string(true, raw, |c| {
                                    if c == '\'' {
                                        qs += 1;
                                        qs == 3
                                    } else {
                                        qs = 0;
                                        false
                                    }
                                }))
                            } else {
                                Some(self.string(false, raw, |c| c == '\''))
                            }
                        }
                        Token::OpeningCurly | Token::OpeningRound | Token::OpeningSquare => {
                            self.parens += 1;
                            self.wrap(token)
                        }
                        Token::ClosingCurly | Token::ClosingRound | Token::ClosingSquare => {
                            self.parens -= 1;
                            self.wrap(token)
                        }
                        _ => self.wrap(token),
                    },
                }
            };
        }
    }
}

#[derive(Debug, Clone, Eq, PartialEq, derive_more::Display)]
pub struct TokenInt(pub(crate) StarlarkInt);

/// All token that can be generated by the lexer
#[derive(Logos, Debug, Clone, PartialEq)]
pub enum Token {
    #[regex(" +", logos::skip)] // Whitespace
    #[token("\\\n", logos::skip)] // Escaped newline
    #[token("\\\r\n", logos::skip)] // Escaped newline (Windows line ending)
    #[regex(r#"#[^\n]*"#, logos::skip)] // Comments
    #[error]
    Error,

    #[regex("\t+")] // Tabs (might be an error)
    Tabs,

    // Indentation block & meaningful spaces
    Indent, // New indentation block
    Dedent, // Leaving an indentation block
    #[regex(r"(\r)?\n")]
    Newline, // Newline outside a string

    // Some things the lexer can't deal with well, so we step in and generate
    // things ourselves
    #[token("'")]
    #[token("r'")]
    RawSingleQuote,
    #[token("\"")]
    #[token("r\"")]
    RawDoubleQuote,

    #[regex(
        "as|\
        assert|\
        async|\
        await|\
        class|\
        del|\
        except|\
        finally|\
        from|\
        global|\
        import|\
        is|\
        nonlocal|\
        raise|\
        try|\
        while|\
        with|\
        yield"
    )]
    Reserved, // One of the reserved keywords

    #[regex(
        "[a-zA-Z_][a-zA-Z0-9_]*"
    , |lex| lex.slice().to_owned())]
    Identifier(String), // An identifier

    #[regex("[0-9]+")]
    RawDecInt,
    #[regex("0[xX][A-Fa-f0-9]+")]
    RawHexInt,
    #[regex("0[bB][01]+")]
    RawBinInt,
    #[regex("0[oO][0-7]+")]
    RawOctInt,

    Int(TokenInt), // An integer literal (123, 0x1, 0b1011, 0o755, ...)

    #[regex("[0-9]+\\.[0-9]*([eE][-+]?[0-9]+)?", |lex| lex.slice().parse::<f64>())]
    #[regex("[0-9]+[eE][-+]?[0-9]+", |lex| lex.slice().parse::<f64>())]
    #[regex("\\.[0-9]+([eE][-+]?[0-9]+)?", |lex| lex.slice().parse::<f64>())]
    Float(f64), // A float literal (3.14, .3, 1e6, 0.)

    String(String), // A string literal

    // Keywords
    #[token("and")]
    And,
    #[token("break")]
    Break,
    #[token("continue")]
    Continue,
    #[token("def")]
    Def,
    #[token("elif")]
    Elif,
    #[token("else")]
    Else,
    #[token("for")]
    For,
    #[token("if")]
    If,
    #[token("in")]
    In,
    #[token("lambda")]
    Lambda,
    #[token("load")]
    Load,
    #[token("not")]
    Not,
    #[token("or")]
    Or,
    #[token("pass")]
    Pass,
    #[token("return")]
    Return,
    // Symbols
    #[token(",")]
    Comma,
    #[token(";")]
    Semicolon,
    #[token(":")]
    Colon,
    #[token("+=")]
    PlusEqual,
    #[token("-=")]
    MinusEqual,
    #[token("*=")]
    StarEqual,
    #[token("/=")]
    SlashEqual,
    #[token("//=")]
    SlashSlashEqual,
    #[token("%=")]
    PercentEqual,
    #[token("==")]
    EqualEqual,
    #[token("!=")]
    BangEqual,
    #[token("<=")]
    LessEqual,
    #[token(">=")]
    GreaterEqual,
    #[token("**")]
    StarStar,
    #[token("->")]
    MinusGreater,
    #[token("=")]
    Equal,
    #[token("<")]
    LessThan,
    #[token(">")]
    GreaterThan,
    #[token("-")]
    Minus,
    #[token("+")]
    Plus,
    #[token("*")]
    Star,
    #[token("%")]
    Percent,
    #[token("/")]
    Slash,
    #[token("//")]
    SlashSlash,
    #[token(".")]
    Dot,
    #[token("&")]
    Ampersand,
    #[token("|")]
    Pipe,
    #[token("^")]
    Caret,
    #[token("<<")]
    LessLess,
    #[token(">>")]
    GreaterGreater,
    #[token("~")]
    Tilde,
    #[token("&=")]
    AmpersandEqual,
    #[token("|=")]
    PipeEqual,
    #[token("^=")]
    CaretEqual,
    #[token("<<=")]
    LessLessEqual,
    #[token(">>=")]
    GreaterGreaterEqual,

    // Brackets
    #[token("[")]
    OpeningSquare,
    #[token("{")]
    OpeningCurly,
    #[token("(")]
    OpeningRound,
    #[token("]")]
    ClosingSquare,
    #[token("}")]
    ClosingCurly,
    #[token(")")]
    ClosingRound,
}

impl Token {
    /// Used for testing
    pub(crate) fn unlex(&self) -> String {
        match self {
            Token::Indent => "\t".to_owned(),
            Token::Newline => "\n".to_owned(),
            Token::Dedent => "#dedent".to_owned(),
            Token::String(x) => {
                // The Rust {:?} is unstable, so changes between versions,
                // instead use the JSON standard for string escapes.
                // Reuse the StarlarkValue implementation since it's close to hand.
                serde_json::to_string(x).unwrap()
            }
            _ => {
                let s = self.to_string();
                // Out display is often: keyword 'lambda'
                // so strip out the bit in single quotes
                let first = s.find('\'');
                match first {
                    Some(first) if s.ends_with('\'') && first != s.len() - 1 => {
                        s[first + 1..s.len() - 1].to_owned()
                    }
                    _ => s,
                }
            }
        }
    }
}

impl Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Token::Error => write!(f, "lexical error"),
            Token::Indent => write!(f, "new indentation block"),
            Token::Dedent => write!(f, "end of indentation block"),
            Token::Newline => write!(f, "new line"),
            Token::And => write!(f, "keyword 'and'"),
            Token::Else => write!(f, "keyword 'else'"),
            Token::Load => write!(f, "keyword 'load'"),
            Token::Break => write!(f, "keyword 'break'"),
            Token::For => write!(f, "keyword 'for'"),
            Token::Not => write!(f, "keyword 'not'"),
            Token::Continue => write!(f, "keyword 'continue'"),
            Token::If => write!(f, "keyword 'if'"),
            Token::Or => write!(f, "keyword 'or'"),
            Token::Def => write!(f, "keyword 'def'"),
            Token::In => write!(f, "keyword 'in'"),
            Token::Pass => write!(f, "keyword 'pass'"),
            Token::Elif => write!(f, "keyword 'elif'"),
            Token::Return => write!(f, "keyword 'return'"),
            Token::Lambda => write!(f, "keyword 'lambda'"),
            Token::Comma => write!(f, "symbol ','"),
            Token::Semicolon => write!(f, "symbol ';'"),
            Token::Colon => write!(f, "symbol ':'"),
            Token::PlusEqual => write!(f, "symbol '+='"),
            Token::MinusEqual => write!(f, "symbol '-='"),
            Token::StarEqual => write!(f, "symbol '*='"),
            Token::SlashEqual => write!(f, "symbol '/='"),
            Token::SlashSlashEqual => write!(f, "symbol '//='"),
            Token::PercentEqual => write!(f, "symbol '%='"),
            Token::EqualEqual => write!(f, "symbol '=='"),
            Token::BangEqual => write!(f, "symbol '!='"),
            Token::LessEqual => write!(f, "symbol '<='"),
            Token::GreaterEqual => write!(f, "symbol '>='"),
            Token::StarStar => write!(f, "symbol '**'"),
            Token::MinusGreater => write!(f, "symbol '->'"),
            Token::Equal => write!(f, "symbol '='"),
            Token::LessThan => write!(f, "symbol '<'"),
            Token::GreaterThan => write!(f, "symbol '>'"),
            Token::Minus => write!(f, "symbol '-'"),
            Token::Plus => write!(f, "symbol '+'"),
            Token::Star => write!(f, "symbol '*'"),
            Token::Percent => write!(f, "symbol '%'"),
            Token::Slash => write!(f, "symbol '/'"),
            Token::SlashSlash => write!(f, "symbol '//'"),
            Token::Dot => write!(f, "symbol '.'"),
            Token::Ampersand => write!(f, "symbol '&'"),
            Token::Pipe => write!(f, "symbol '|'"),
            Token::Caret => write!(f, "symbol '^'"),
            Token::LessLess => write!(f, "symbol '<<'"),
            Token::GreaterGreater => write!(f, "symbol '>>'"),
            Token::Tilde => write!(f, "symbol '~'"),
            Token::AmpersandEqual => write!(f, "symbol '&='"),
            Token::PipeEqual => write!(f, "symbol '|='"),
            Token::CaretEqual => write!(f, "symbol '^='"),
            Token::LessLessEqual => write!(f, "symbol '<<='"),
            Token::GreaterGreaterEqual => write!(f, "symbol '>>='"),
            Token::OpeningSquare => write!(f, "symbol '['"),
            Token::OpeningCurly => write!(f, "symbol '{{'"),
            Token::OpeningRound => write!(f, "symbol '('"),
            Token::ClosingSquare => write!(f, "symbol ']'"),
            Token::ClosingCurly => write!(f, "symbol '}}'"),
            Token::ClosingRound => write!(f, "symbol ')'"),
            Token::Reserved => write!(f, "reserved keyword"),
            Token::Identifier(s) => write!(f, "identifier '{}'", s),
            Token::Int(i) => write!(f, "integer literal '{}'", i),
            Token::RawDecInt => write!(f, "decimal integer literal"),
            Token::RawHexInt => write!(f, "hexadecimal integer literal"),
            Token::RawOctInt => write!(f, "octal integer literal"),
            Token::RawBinInt => write!(f, "binary integer literal"),
            Token::Float(n) => write!(f, "float literal '{}'", n),
            Token::String(s) => write!(f, "string literal '{}'", s),
            Token::RawSingleQuote => write!(f, "starting '"),
            Token::RawDoubleQuote => write!(f, "starting \""),
            Token::Tabs => Ok(()),
        }
    }
}

impl<'a> Iterator for Lexer<'a> {
    type Item = Lexeme;

    fn next(&mut self) -> Option<Self::Item> {
        self.next()
    }
}