The Challenge
-------------
You'll need to implement a simple lexer type. It should take in an input string through the constructor (or the parameter, for Javascript), and break it up into typed-tokens (in python, C# and Java, you'll have to manage `null/None` input too, resulting in the same behavior than an empty string). You'll need to implement the necessary methods (aaccording to your language) to make the Simplexer object behave like an iterator, Meaning that it returns a token (assuming one is available) object each time it a `next` (`Current` field in C#) method would be called. If no tokens are available, an exception should be thrown (idealy: `StopIteration` in python, `InvalidOperationException` in C# and `NoSuchElementException` in Java).

Tokens are represented by Token objects, which define two properties as strings: `text`, and `type`. Constructor is `Token(text, type)`.

**C# Notes**:
`Iterator` is an extension of `IEnumerator` with default implementations for `Reset()`, `Dispose()` and `IEnumerator.Current` as these are not need to pass the challenge. You only need to override `MoveNext()` and `Current { get; }`.

Token Types
-----------
There are 7 tokens types that your lexer will need to produce: `identifier`, `string`, `integer`, `boolean`, `keyword`, `operator`, and `whitespace`. To create the token, you'd need to pass in the token value (the text) and the token type as strings, so for example, a simple integer token could be created with `new Token("1", "integer")` (Note: no default values or default constructor are provided, so use `new Token("","")` if you want a default Token object).

Token Grammar
--------------
Here's a table of the grammars for the various token types:
```
integer : Any sequence of one or more digits.

boolean : true or false.

string : Any sequence of characters surrounded by "double quotes".

operator : The characters +, -, *, /, %, (, ), and =.

keyword : The following are keywords: if, else, for, while, return, func, and break.

whitespace : Matches standard whitespace characters (space, newline, tab, etc.)
Consecutive whitespace characters should be matched together.

identifier : Any sequence of alphanumber characters, as well as underscore and dollar sign,
and which doesn't start with a digit. Make sure that keywords aren't matched as identifiers!
```

Source: codewars | Difficulty: MEDIUM
Tags: String algorithms

Starter code: def __init__(self, expression):
	

Solutions:
import re

class Simplexer(object):
	ORDERED_TOKENS = [{'type': 'integer', 'reg': '\\d+'}, {'type': 'boolean', 'reg': 'true|false'}, {'type': 'string', 'reg': '\\".*\\"'}, {'type': 'operator', 'reg': '[-+*/%\\)\\(=]'}, {'type': 'keyword', 'reg': 'if|else|for|while|return|func|break'}, {'type': 'whitespace', 'reg': '\\s+'}, {'type': 'identifier', 'reg': '[$_a-zA-Z][$\\w]*'}]
	PATTERN = re.compile('|'.join(('(?P<{}>{})'.format(dct['type'], dct['reg']) for dct in ORDERED_TOKENS)))

	def __init__(self, s):
		self.iterable = Simplexer.PATTERN.finditer(s)

	def __iter__(self):
		return self

	def __next__(self):
		for m in self.iterable:
			for (k, s) in m.groupdict().items():
				if s is None:
					continue
				return Token(s, k)
		raise StopIteration


import re

class Simplexer:
	regex = re.compile('|'.join(('(?P<{}>{})'.format(token_type, regex) for (token_type, regex) in (('integer', '\\d+'), ('boolean', '\\b(?:true|false)\\b'), ('string', '"[^"]*"'), ('operator', '[*/%()=+-]'), ('keyword', '\\b(?:if|else|for|while|return|func|break)\\b'), ('whitespace', '\\s+'), ('identifier', '[a-zA-Z_$][a-zA-Z0-9_$]*')))))

	def __init__(self, expression):
		self.iter = self.regex.finditer(expression)

	def __iter__(self):
		return self

	def __next__(self):
		match = next(self.iter)
		return Token(match.group(0), match.lastgroup)


from re import compile as reCompile
PATTERN = reCompile('(?P<integer>\\d+)|(?P<boolean>true|false)|(?P<string>".*")|(?P<operator>[+*/%()=-])|(?P<keyword>if|else|for|while|return|func|break)|(?P<whitespace>\\s+)|(?P<identifier>[a-zA-Z_$][a-zA-Z0-9_$]*)')

class Simplexer:

	def __init__(self, s):
		self.s = s
		self.pos = 0

	def __iter__(self):
		return self

	def __call__(self):
		return self

	def __next__(self):
		m = PATTERN.match(self.s, self.pos)
		if not m:
			raise StopIteration()
		self.pos = m.end()
		for (name, value) in m.groupdict().items():
			if value:
				return Token(value, name)

	@classmethod
	def simplex(cls, s):
		return cls(s)


from re import findall

class Simplexer(object):

	def __init__(self, expression):
		self.current = 0
		self.result = []
		for x in findall('(".*"|true|false|\\d+|[a-zA-Z]+|[+\\-()\\/*=]|\\s+)', expression):
			if x[0] == '"':
				type = 'string'
			elif x.isdigit():
				type = 'integer'
			elif x.isspace():
				type = 'whitespace'
			elif x in '+-()/*=':
				type = 'operator'
			elif x in ('true', 'false'):
				type = 'boolean'
			elif x in ('if', 'break', 'return', 'for', 'else'):
				type = 'keyword'
			else:
				type = 'identifier'
			self.result.append(Token(x, type))

	def __iter__(self):
		return self

	def __next__(self):
		if self.current == len(self.result):
			raise StopIteration
		res = self.result[self.current]
		self.current += 1
		return res


import re

class Simplexer(object):

	def __init__(self, expression):
		(r, finded) = ([], [])
		pattern = [('\\d+', 'integer'), ('\\btrue\\b|\\bfalse\\b', 'boolean'), ('".*"', 'string'), ('[-+*/%().=]', 'operator'), ('if|else|for|while|return|func|break', 'keyword'), ('\\s+', 'whitespace'), ('[a-zA-Z0-9_$]+', 'identifier')]

		def do(a, b, s=0):
			for i in re.finditer(a, expression):
				r1 = not any((k <= i.start() <= l for (k, l) in finded))
				if s and (not i.group()[0].isdigit()) and r1 or r1:
					finded.append([i.start(), i.end() - 1])
					r.append([i.group(), i.start(), b])
		for (i, j) in pattern:
			do(i, j, 0 if j != 'identifier' else 1)
		self.r = [Token(i[0], i[2]) for i in sorted(r, key=lambda x: x[1])]
		(self.l, self.n) = (len(self.r), -1)

	def __iter__(self):
		return self

	def __next__(self):
		try:
			self.n += 1
			return self.r[self.n]
		except IndexError:
			raise StopIteration


class Simplexer(object):

	def __init__(self, expression):
		self.tokens = []
		l = list(expression)[::-1]
		while l:
			t = l.pop()
			if t in '+-*/%()=':
				ttype = 'operator'
			elif t.isspace():
				ttype = 'whitespace'
				while l and l[-1].isspace():
					t += l.pop()
			elif t.isdigit():
				ttype = 'integer'
				while l and l[-1].isdigit():
					t += l.pop()
			elif t == '"':
				ttype = 'string'
				while l and l[-1] != '"':
					t += l.pop()
				t += l.pop()
			else:
				while l and (not l[-1].isspace()) and (not l[-1] in '+-*/%()='):
					t += l.pop()
				ttype = 'boolean' if t in 'true false' else 'keyword' if t in 'if else for while return func and break'.split() else 'identifier'
			self.tokens.append(Token(t, ttype))
		self.tokens = self.tokens[::-1]

	def __iter__(self):
		return iter(self.tokens[::-1])

	def __next__(self):
		if self.tokens:
			return self.tokens.pop()
		raise StopIteration()


from itertools import chain

class Simplexer(object):
	BOOLS = ['true', 'false']
	KEYWORDS = ['if', 'else', 'for', 'while', 'return', 'func', 'break']
	OPERATORS = '+-*/%()='
	SPACE = ' \n\t\\c'
	NUMBER = '0123456789'
	CHAR = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$'

	def __init__(self, expression):
		self.expression = expression
		self.__data = iter(expression)

	def __iter__(self):
		self.__data = iter(self.expression)
		return self

	def __next__(self):
		token = self._nextc()
		if token in self.OPERATORS:
			return Token(token, 'operator')
		if token in self.SPACE:
			while self._peekc() in self.SPACE:
				token += self._nextc()
			return Token(token, 'whitespace')
		if token == '"':
			token += self._nextc()
			while token[-1] != '"':
				token += self._nextc()
			return Token(token, 'string')
		if token in self.NUMBER:
			while self._peekc() in self.NUMBER:
				token += self._nextc()
			return Token(token, 'integer')
		if token in self.CHAR:
			while self._peekc() in self.CHAR + self.NUMBER:
				token += self._nextc()
			if token in self.BOOLS:
				return Token(token, 'boolean')
			if token in self.KEYWORDS:
				return Token(token, 'keyword')
			return Token(token, 'identifier')

	def _nextc(self):
		return next(self.__data)

	def _peekc(self):
		try:
			char = next(self.__data)
			self.__data = chain([char], self.__data)
		except:
			char = 'END'
		return char


class Simplexer:

	def __init__(self, expression):
		self.n = 0
		self.keywords = ['if', 'else', 'for', 'while', 'return', 'func', 'break']
		self.whitespace_chars = [' ', '\t', '\r', '\n', '\x0b', '\x0c']
		self.expression = expression
		self.work_dict = {}
		self.__build()
		self.tokens = [Token(text=self.work_dict[x]['text'], type=self.work_dict[x]['type']) for x in sorted(self.work_dict.keys(), key=lambda x: int(x.split(',')[0]))]

	def __iter__(self):
		for x in self.tokens:
			yield x

	def __next__(self):
		if len(self.tokens) == 0:
			raise StopIteration
		if self.n < len(self.tokens):
			self.n += 1
			return self.tokens[self.n - 1]
		else:
			raise StopIteration

	def __str__(self):
		return ', '.join([str(t) for t in self.tokens])

	def __repr__(self):
		return str(self)

	def __eq__(self, other):
		if isinstance(other, Simplexer):
			if len(self) != len(other):
				return False
			for t in self.tokens:
				found = False
				for t2 in other.tokens:
					if t.text == t2.text and t.type == t2.type:
						found = True
						break
				if not found:
					return False
			return True
		elif isinstance(other, list):
			for t in self.tokens:
				found = False
				for t2 in other:
					if t.text == t2.text and t.type == t2.type:
						found = True
						break
				if not found:
					return False
			return True
		else:
			return False

	def __build(self):
		self.__handle_integers()
		self.__handle_booleans()
		self.__handle_strings()
		self.__handle_operators()
		self.__handle_keywords()
		self.__handle_whitespaces()
		self.__handle_identifiers()

	def __handle_integers(self):
		i = 0
		count = 0
		while i < len(self.expression) and count < len(self.expression):
			if self.expression[i].isdigit():
				c = self.expression[i]
				numbers = [c]
				count = i + 1
				while c.isdigit() and count < len(self.expression):
					c = self.expression[count]
					if not c.isdigit():
						break
					numbers.append(c)
					count += 1
				self.work_dict[f'{i},{count}'] = {'text': ''.join(numbers), 'type': 'integer'}
			else:
				count += 1
			i = count

	def __handle_booleans(self):
		import re
		true_idxes = [f'{m.start()},{m.end()}' for m in re.finditer('true', self.expression)]
		for t in true_idxes:
			self.work_dict[t] = {'text': 'true', 'type': 'boolean'}
		false_idxes = [f'{m.start()},{m.end()}' for m in re.finditer('false', self.expression)]
		for t in false_idxes:
			self.work_dict[t] = {'text': 'false', 'type': 'boolean'}

	def __handle_strings(self):
		import re
		pattern = '".*?"'
		idxes = [f'{m.start()},{m.end()}' for m in re.finditer(pattern, self.expression)]
		for t in idxes:
			(start, end) = [int(elem) for elem in t.split(',')]
			self.work_dict[t] = {'text': self.expression[start:end], 'type': 'string'}

	def __handle_operators(self):
		for i in range(len(self.expression)):
			if self.expression[i] in ['+', '-', '*', '/', '%', '(', ')', '=']:
				self.work_dict[f'{i},{i + 1}'] = {'text': self.expression[i], 'type': 'operator'}

	def __handle_keywords(self):
		import re
		for word in self.keywords:
			idxes = [f'{m.start()},{m.end()}' for m in re.finditer(word, self.expression)]
			for t in idxes:
				(start, end) = [int(elem) for elem in t.split(',')]
				self.work_dict[t] = {'text': self.expression[start:end], 'type': 'keyword'}
		return

	def __handle_whitespaces(self):
		i = 0
		count = 0
		while i < len(self.expression) and count < len(self.expression):
			if self.expression[i] in self.whitespace_chars:
				c = self.expression[i]
				spaces = [c]
				count = i + 1
				while c in self.whitespace_chars and count < len(self.expression):
					c = self.expression[count]
					if c not in self.whitespace_chars:
						break
					spaces.append(c)
					count += 1
				self.work_dict[f'{i},{count}'] = {'text': ''.join(spaces), 'type': 'whitespace'}
			else:
				count += 1
			i = count

	def __handle_identifiers(self):
		sorted_keys = sorted(self.work_dict.keys(), key=lambda x: int(x.split(',')[0]))
		if len(sorted_keys) == 0:
			if len(self.expression) == 1:
				self.work_dict['0,1'] = {'text': self.expression, 'type': 'identifier'}
			return
		start = int(sorted_keys[0].split(',')[0])
		if start != 0:
			self.work_dict[f'0,{start}'] = {'text': self.expression[0:start], 'type': 'identifier'}
		end = int(sorted_keys[-1].split(',')[1])
		if end != len(self.expression):
			self.work_dict[f'{end},{len(self.expression)}'] = {'text': self.expression[end:len(self.expression)], 'type': 'identifier'}
		prev_end = int(sorted_keys[0].split(',')[1])
		for key in sorted_keys:
			(start, end) = [int(x) for x in key.split(',')]
			if start != prev_end and self.expression[prev_end:start] != '':
				self.work_dict[f'{prev_end},{start}'] = {'text': self.expression[prev_end:start], 'type': 'identifier'}
			prev_end = end

class Token:

	def __init__(self, text, type):
		self.text = text
		self.type = type

	def __str__(self):
		return f'Token("{self.text}", "{self.type}")'

	def __repr__(self):
		return str(self)

	def __eq__(self, other):
		if isinstance(other, Token):
			return self.text == other.text and self.type == other.type
		return False


TOKENS_TYPE = {'+': 'operator', '-': 'operator', '*': 'operator', '/': 'operator', '%': 'operator', '(': 'operator', ')': 'operator', '=': 'operator', 'true': 'boolean', 'false': 'boolean', 'if': 'keyword', 'for': 'keyword', 'while': 'keyword', 'else': 'keyword', 'return': 'keyword', 'func': 'keyword', 'break': 'keyword', ' ': 'whitespace'}

class Token:

	def __init__(self, text: str, type: str):
		self.text = text
		self.type = type

	def __eq__(self, other):
		return self.text == other.text and self.type == other.type

	def __repr__(self):
		return f"<'{self.text}': {self.type}>"

class Simplexer:

	def __init__(self, expression):
		self.exp = expression
		self.num = 0
		self.parsed_exp = self._parse()

	@staticmethod
	def _get_split_chars(char_types: list):
		split_types = []
		temp_type = (0, char_types[0][-1])
		for i in range(len(char_types)):
			if char_types[i][-1] != temp_type[-1]:
				split_types.append(char_types[temp_type[0]:i])
				temp_type = (i, char_types[i][-1])
		if not split_types:
			split_types.append([char_data for char_data in char_types])
		else:
			split_types.append(char_types[-1][0])
		return split_types

	def _parse(self):
		if self.exp:
			char_types = []
			for char in self.exp:
				char_types.append((char, self.get_seq_type(char)))
			result = []
			split_chars = self._get_split_chars(char_types)
			for pack in split_chars:
				seq = ''
				if pack[0][-1] != 'operator':
					for char_data in pack:
						seq += char_data[0]
					seq_type = self.get_seq_type(seq)
					result.append(Token(seq, seq_type))
				else:
					result.extend([Token(i[0], 'operator') for i in pack])
			return result

	@staticmethod
	def get_seq_type(s: str) -> str:
		if s.isdigit():
			return 'integer'
		elif s in TOKENS_TYPE:
			return TOKENS_TYPE[s]
		elif not s.split():
			return 'whitespace'
		elif len(s) > 1 and s[0] in ("'", '"') and (s[-1] in ("'", '"')):
			return 'string'
		return 'identifier'

	def __iter__(self):
		return self

	def __next__(self):
		num = self.num
		self.num += 1
		if self.num <= len(self.parsed_exp):
			return self.parsed_exp[num]
		raise StopIteration


class Simplexer:

	def __init__(self, expression):
		self.expression = expression
		self.pos = 0

	def __iter__(self):
		return self

	def __next__(self):
		length = len(self.expression)
		start = self.pos
		if self.pos == length:
			raise StopIteration
		if self.expression[start] == '"':
			self.pos = start + 1
			while self.pos < length and self.expression[self.pos] != '"':
				self.pos += 1
			self.pos += 1
			return Token(self.expression[start:self.pos], 'string')
		if self.expression[start] in '+-*/%()=':
			self.pos += 1
			return Token(self.expression[start], 'operator')
		for (typ, func) in scans:
			if func(self.expression[start]):
				self.pos = start + 1
				while self.pos < length and func(self.expression[self.pos]):
					self.pos += 1
				value = self.expression[start:self.pos]
				if typ == 'identifier':
					if value in ('true', 'false'):
						typ = 'boolean'
					elif value in keywords:
						typ = 'keyword'
				return Token(value, typ)
keywords = {'if', 'else', 'for', 'while', 'return', 'func', 'break'}
scans = (('integer', lambda c: c.isdigit()), ('whitespace', lambda c: c.isspace()), ('identifier', lambda c: c.isalnum() or c in '$_'))