--
local line_verbs = {
	CR = 0, LF = 1, CRLF = 2, ANYRLF = 3, ANY = 4, NUL = 5,
};
local function is_newline(str_arr, i, verb_flags)
	local line_verb_n = verb_flags.newline;
	local chr = str_arr[i];
	if line_verb_n == 0 then
		-- carriage return
		return chr == 0x0D;
	elseif line_verb_n == 2 then
		-- carriage return followed by line feed
		return chr == 0x0A and str_arr[i - 1] == 0x20;
	elseif line_verb_n == 3 then
		-- any of the above
		return chr == 0x0A or chr == 0x0D;
	elseif line_verb_n == 4 then
		-- any of Unicode newlines
		return chr == 0x0A or chr == 0x0B or chr == 0x0C or chr == 0x0D or chr == 0x85 or chr == 0x2028 or chr == 0x2029;
	elseif line_verb_n == 5 then
		-- null
		return chr == 0;
	end;
	-- linefeed
	return chr == 0x0A;
end;


local function tkn_char_match(tkn_part, str_arr, i, flags, verb_flags)
	local chr = str_arr[i];
	if not chr then
		return false;
	elseif flags.ignoreCase and chr >= 0x61 and chr <= 0x7A then
		chr -= 0x20;
	end;
	if type(tkn_part) == "number" then
		return tkn_part == chr;
	elseif tkn_part[1] == "charset" then
		for _, v in ipairs(tkn_part[3]) do
			if tkn_char_match(v, str_arr, i, flags, verb_flags) then
				return not tkn_part[2];
			end;
		end;
		return tkn_part[2];
	elseif tkn_part[1] == "range" then
		return chr >= tkn_part[2] and chr <= tkn_part[3] or flags.ignoreCase and chr >= 0x41 and chr <= 0x5A and (chr + 0x20) >= tkn_part[2] and (chr + 0x20) <= tkn_part[3];
	elseif tkn_part[1] == "class" then
		local char_class = tkn_part[2];
		local negate = tkn_part[3];
		local match = false;
		-- if and elseifs :(
		-- Might make these into tables in the future
		if char_class == "xdigit" then
			match = chr >= 0x30 and chr <= 0x39 or chr >= 0x41 and chr <= 0x46 or chr >= 0x61 and chr <= 0x66;
		elseif char_class == "ascii" then
			match = chr <= 0x7F;
		-- cannot be accessed through POSIX classes
		elseif char_class == "vertical_tab" then
			match = chr >= 0x0A and chr <= 0x0D or chr == 0x2028 or chr == 0x2029;
		--
		elseif flags.unicode then
			local current_category = u_categories[chr] or 'Cn';
			local first_category = current_category:sub(1, 1);
			if char_class == "alnum" then
				match = first_category == 'L' or current_category == 'Nl' or current_category == 'Nd';
			elseif char_class == "alpha" then
				match = first_category == 'L' or current_category == 'Nl';
			elseif char_class == "blank" then
				match = current_category == 'Zs' or chr == 0x09;
			elseif char_class == "cntrl" then
				match = current_category == 'Cc';
			elseif char_class == "digit" then
				match = current_category == 'Nd';
			elseif char_class == "graph" then
				match = first_category ~= 'P' and first_category ~= 'C';
			elseif char_class == "lower" then
				match = current_category == 'Ll';
			elseif char_class == "print" then
				match = first_category ~= 'C';
			elseif char_class == "punct" then
				match = first_category == 'P';
			elseif char_class == "space" then
				match = first_category == 'Z' or chr >= 0x09 and chr <= 0x0D;
			elseif char_class == "upper" then
				match = current_category == 'Lu';
			elseif char_class == "word" then
				match = first_category == 'L' or current_category == 'Nl' or current_category == 'Nd' or current_category == 'Pc';
			end;
		elseif char_class == "alnum" then
			match = chr >= 0x30 and chr <= 0x39 or chr >= 0x41 and chr <= 0x5A or chr >= 0x61 and chr <= 0x7A;
		elseif char_class == "alpha" then
			match = chr >= 0x41 and chr <= 0x5A or chr >= 0x61 and chr <= 0x7A;
		elseif char_class == "blank" then
			match = chr == 0x09 or chr == 0x20;
		elseif char_class == "cntrl" then
			match = chr <= 0x1F or chr == 0x7F;
		elseif char_class == "digit" then
			match = chr >= 0x30 and chr <= 0x39;
		elseif char_class == "graph" then
			match = chr >= 0x21 and chr <= 0x7E;
		elseif char_class == "lower" then
			match = chr >= 0x61 and chr <= 0x7A;
		elseif char_class == "print" then
			match = chr >= 0x20 and chr <= 0x7E;
		elseif char_class == "punct" then
			match = class_ascii_punct[chr];
		elseif char_class == "space" then
			match = chr >= 0x09 and chr <= 0x0D or chr == 0x20;
		elseif char_class == "upper" then
			match = chr >= 0x41 and chr <= 0x5A;
		elseif char_class == "word" then
			match = chr >= 0x30 and chr <= 0x39 or chr >= 0x41 and chr <= 0x5A or chr >= 0x61 and chr <= 0x7A or chr == 0x5F;
		end;
		if negate then
			return not match;
		end;
		return match;
	elseif tkn_part[1] == "category" then
		local chr_category = u_categories[chr] or 'Cn';
		local category_v = tkn_part[3];
		local category_len = #category_v;
		if category_len == 3 then
			local match = false;
			if category_v == "Xan" or category_v == "Xwd" then
				match = chr_category:find("^[LN]") or category_v == "Xwd" and chr == 0x5F;
			elseif category_v == "Xps" or category_v == "Xsp" then
				match = chr_category:sub(1, 1) == 'Z' or chr >= 0x09 and chr <= 0x0D;
			elseif category_v == "Xuc" then
				match = tkn_char_match(xuc_chr, str_arr, i, flags, verb_flags);
			end;
			if tkn_part[2] then
				return not match;
			end
			return match;
		elseif chr_category:sub(1, category_len) == category_v then
			return not tkn_part[2];
		end;
		return tkn_part[2];
	elseif tkn_part[1] == 0x2E then
		return flags.dotAll or not is_newline(str_arr, i, verb_flags);
	elseif tkn_part[1] == 0x4E then
		return not is_newline(str_arr, i, verb_flags);
	elseif tkn_part[1] == 0x52 then
		if verb_flags.newline_seq == 0 then
			-- CR, LF or CRLF
			return chr == 0x0A or chr == 0x0D;
		end;
		-- any unicode newline
		return chr == 0x0A or chr == 0x0B or chr == 0x0C or chr == 0x0D or chr == 0x85 or chr == 0x2028 or chr == 0x2029;
	end;
	return false;
end;

local function find_alternation(token, i, count)
	while true do
		local v = token[i];
		local is_table = type(v) == "table";
		if v == alternation then
			return i, count;
		elseif is_table and v[1] == 0x28 then
			if count then
				count += v.count;
			end;
			i = v[3];
		elseif is_table and v[1] == "quantifier" and type(v[5]) == "table" and v[5][1] == 0x28 then
			if count then
				count += v[5].count;
			end;
			i = v[5][3];
		elseif not v or is_table and v[1] == 0x29 then
			return nil, count;
		elseif count then
			if is_table and v[1] == "quantifier" then
				count += v[3];
			else
				count += 1;
			end;
		end;
		i += 1;
	end;
end;

local function re_rawfind(token, str_arr, init, flags, verb_flags, as_bool)
	local tkn_i, str_i, start_i = 0, init, init;
	local states = { };
	while tkn_i do
		if tkn_i == 0 then
			tkn_i += 1;
			local next_alt = find_alternation(token, tkn_i);
			if next_alt then
				table.insert(states, 1, { "alternation", next_alt, str_i });
			end;
			continue;
		end;
		local ctkn = token[tkn_i];
		local tkn_type = type(ctkn) == "table" and ctkn[1];
		if not ctkn then
			break;
		elseif ctkn == "ACCEPT" then
			local not_lookaround = true;
			local close_i = tkn_i;
			repeat
				close_i += 1;
				local is_table = type(token[close_i]) == "table";
				local close_i_tkn = token[close_i];
				if is_table and (close_i_tkn[1] == 0x28 or close_i_tkn[1] == "quantifier" and type(close_i_tkn[5]) == "table" and close_i_tkn[5][1] == 0x28) then
					close_i = close_i_tkn[1] == "quantifier" and close_i_tkn[5][3] or close_i_tkn[3];
				elseif is_table and close_i_tkn[1] == 0x29 and (close_i_tkn[4] == 0x21 or close_i_tkn[4] == 0x3D) then
					not_lookaround = false;
					tkn_i = close_i;
					break;
				end;
			until not close_i_tkn;
			if not_lookaround then
				break;
			end;
		elseif ctkn == "PRUNE" or ctkn == "SKIP" then
			table.insert(states, 1, { ctkn, str_i });
			tkn_i += 1;
		elseif tkn_type == 0x28 then
			table.insert(states, 1, { "group", tkn_i, str_i, nil, ctkn[2], ctkn[3], ctkn[4] });
			tkn_i += 1;
			local next_alt, count = find_alternation(token, tkn_i, (ctkn[4] == 0x21 or ctkn[4] == 0x3D) and ctkn[5] and 0);
			if next_alt then
				table.insert(states, 1, { "alternation", next_alt, str_i });
			end;
			if count then
				str_i -= count;
			end;
		elseif tkn_type == 0x29 and ctkn[4] ~= 0x21 then
			if ctkn[4] == 0x21 or ctkn[4] == 0x3D then
				while true do
					local selected_match_start;
					local selected_state = table.remove(states, 1);
					if selected_state[1] == "group" and selected_state[2] == ctkn[3] then
						if (ctkn[4] == 0x21 or ctkn[4] == 0x3D) and not ctkn[5] then
							str_i = selected_state[3];
						end;
						if selected_match_start then
							table.insert(states, 1, selected_match_start);
						end;
						break;
					elseif selected_state[1] == "matchStart" and not selected_match_start and ctkn[4] == 0x3D then
						selected_match_start = selected_state;
					end;
				end;
			elseif ctkn[4] == 0x3E then
				repeat
					local selected_state = table.remove(states, 1);
				until not selected_state or selected_state[1] == "group" and selected_state[2] == ctkn[3];
			else
				for i, v in ipairs(states) do
					if v[1] == "group" and v[2] == ctkn[3] then
						if v.jmp then
							-- recursive match
							tkn_i = v.jmp;
						end;
						v[4] = str_i;
						if v[7] == "quantifier" and v[10] + 1 < v[9] then
							if token[ctkn[3]][4] ~= "lazy" or v[10] + 1 < v[8] then
								tkn_i = ctkn[3];
							end;
							local ctkn1 = token[ctkn[3]];
							local new_group = { "group", v[2], str_i, nil, ctkn1[5][2], ctkn1[5][3], "quantifier", ctkn1[2], ctkn1[3], v[10] + 1, v[11], ctkn1[4] };
							table.insert(states, 1, new_group);
							if v[11] then
								table.insert(states, 1, { "alternation", v[11], str_i });
							end;
						end;
						break;
					end;
				end;
			end;
			tkn_i += 1;
		elseif tkn_type == 0x4B then
			table.insert(states, 1, { "matchStart", str_i });
			tkn_i += 1;
		elseif tkn_type == 0x7C then
			local close_i = tkn_i;
			repeat
				close_i += 1;
				local is_table = type(token[close_i]) == "table";
				local close_i_tkn = token[close_i];
				if is_table and (close_i_tkn[1] == 0x28 or close_i_tkn[1] == "quantifier" and type(close_i_tkn[5]) == "table" and close_i_tkn[5][1] == 0x28) then
					close_i = close_i_tkn[1] == "quantifier" and close_i_tkn[5][3] or close_i_tkn[3];
				end;
			until is_table and close_i_tkn[1] == 0x29 or not close_i_tkn;
			if token[close_i] then
				for _, v in ipairs(states) do
					if v[1] == "group" and v[6] == close_i then
						tkn_i = v[6];
						break;
					end;
				end;
			else
				tkn_i = close_i;
			end;
		elseif tkn_type == "recurmatch" then
			table.insert(states, 1, { "group", ctkn[3], str_i, nil, nil, token[ctkn[3]][3], nil, jmp = tkn_i });
			tkn_i = ctkn[3] + 1;
			local next_alt, count = find_alternation(token, tkn_i);
			if next_alt then
				table.insert(states, 1, { "alternation", next_alt, str_i });
			end;
		else
			local match;
			if ctkn == "FAIL" then
				match = false;
			elseif tkn_type == 0x29 then
				repeat
					local selected_state = table.remove(states, 1);
				until selected_state[1] == "group" and selected_state[2] == ctkn[3];
			elseif tkn_type == "quantifier" then
				if type(ctkn[5]) == "table" and ctkn[5][1] == 0x28 then
					local next_alt = find_alternation(token, tkn_i + 1);
					if next_alt then
						table.insert(states, 1, { "alternation", next_alt, str_i });
					end;
					table.insert(states, next_alt and 2 or 1, { "group", tkn_i, str_i, nil, ctkn[5][2], ctkn[5][3], "quantifier", ctkn[2], ctkn[3], 0, next_alt, ctkn[4] });
					if ctkn[4] == "lazy" and ctkn[2] == 0 then
						tkn_i = ctkn[5][3];
					end;
					match = true;
				else
					local start_i, end_i;
					local pattern_count = 1;
					local is_backref = type(ctkn[5]) == "table" and ctkn[5][1] == "backref";
					if is_backref then
						pattern_count = 0;
						local group_n = ctkn[5][2];
						for _, v in ipairs(states) do
							if v[1] == "group" and v[5] == group_n then
								start_i, end_i = v[3], v[4];
								pattern_count = end_i - start_i;
								break;
							end;
						end;
					end;
					local min_max_i = str_i + ctkn[2] * pattern_count;
					local mcount = 0;
					while mcount < ctkn[3] do
						if is_backref then
							if start_i and end_i then
								local org_i = str_i;
								if utf8_sub(str_arr.s, start_i, end_i) ~= utf8_sub(str_arr.s, org_i, str_i + pattern_count) then
									break;
								end;
							else
								break;
							end;
						elseif not tkn_char_match(ctkn[5], str_arr, str_i, flags, verb_flags) then
							break;
						end;
						str_i += pattern_count;
						mcount += 1;
					end;
					match = mcount >= ctkn[2];
					if match and ctkn[4] ~= "possessive" then
						if ctkn[4] == "lazy" then
							min_max_i, str_i = str_i, min_max_i;
						end;
						table.insert(states, 1, { "quantifier", tkn_i, str_i, math.min(min_max_i, str_arr.n + 1), (ctkn[4] == "lazy" and 1 or -1) * pattern_count });
					end;
				end;
			elseif tkn_type == "backref" then
				local start_i, end_i;
				local group_n = ctkn[2];
				for _, v in ipairs(states) do
					if v[1] == "group" and v[5] == group_n then
						start_i, end_i = v[3], v[4];
						break;
					end;
				end;
				if start_i and end_i then
					local org_i = str_i;
					str_i += end_i - start_i;
					match = utf8_sub(str_arr.s, start_i, end_i) == utf8_sub(str_arr.s, org_i, str_i);
				end;
			else
				local chr = str_arr[str_i];
				if tkn_type == 0x24 or tkn_type == 0x5A or tkn_type == 0x7A then
					match = str_i == str_arr.n + 1 or tkn_type == 0x24 and flags.multiline and is_newline(str_arr, str_i + 1, verb_flags) or tkn_type == 0x5A and str_i == str_arr.n and is_newline(str_arr, str_i, verb_flags);
				elseif tkn_type == 0x5E or tkn_type == 0x41 or tkn_type == 0x47 then
					match = str_i == 1 or tkn_type == 0x5E and flags.multiline and is_newline(str_arr, str_i - 1, verb_flags) or tkn_type == 0x47 and str_i == init;
				elseif tkn_type == 0x42 or tkn_type == 0x62 then
					local start_m = str_i == 1 or flags.multiline and is_newline(str_arr, str_i - 1, verb_flags);
					local end_m = str_i == str_arr.n + 1 or flags.multiline and is_newline(str_arr, str_i, verb_flags);
					local w_m = tkn_char_match(ctkn[2], str_arr[str_i - 1], flags) and 0 or tkn_char_match(ctkn[2], chr, flags) and 1;
					if w_m == 0 then
						match = end_m or not tkn_char_match(ctkn[2], chr, flags);
					elseif w_m then
						match = start_m or not tkn_char_match(ctkn[2], str_arr[str_i - 1], flags);
					end;
					if tkn_type == 0x42 then
						match = not match;
					end;
				else
					match = tkn_char_match(ctkn, str_arr, str_i, flags, verb_flags);
					str_i += 1;
				end;
			end;
			if not match then
				while true do
					local prev_type, prev_state = states[1] and states[1][1], states[1];
					if not prev_type or prev_type == "PRUNE" or prev_type == "SKIP" then
						if prev_type then
							table.clear(states);
						end;
						if start_i > str_arr.n then
							if as_bool then
								return false;
							end;
							return nil;
						end;
						start_i = prev_type == "SKIP" and prev_state[2] or start_i + 1;
						tkn_i, str_i = 0, start_i;
						break;
					elseif prev_type == "alternation" then
						tkn_i, str_i = prev_state[2], prev_state[3];
						local next_alt, count = find_alternation(token, tkn_i + 1);
						if next_alt then
							prev_state[2] = next_alt;
						else
							table.remove(states, 1);
						end;
						if count then
							str_i -= count;
						end;
						break;
					elseif prev_type == "group" then
						if prev_state[7] == "quantifier" then
							if prev_state[12] == "greedy" and prev_state[10] >= prev_state[8]
								or prev_state[12] == "lazy" and prev_state[10] < prev_state[9] and not prev_state[13] then
								tkn_i, str_i = prev_state[12] == "greedy" and prev_state[6] or prev_state[2], prev_state[3];
								if prev_state[12] == "greedy" then
									table.remove(states, 1);
									break;
								elseif prev_state[10] >= prev_state[8] then
									prev_state[13] = true;
									break;
								end;
							end;
						elseif prev_state[7] == 0x21 then
							table.remove(states, 1);
							tkn_i, str_i = prev_state[6], prev_state[3];
							break;
						end;
					elseif prev_type == "quantifier" then
						if math.sign(prev_state[4] - prev_state[3]) == math.sign(prev_state[5]) then
							prev_state[3] += prev_state[5];
							tkn_i, str_i = prev_state[2], prev_state[3];
							break;
						end;
					end;
					-- keep match out state and recursive state, can be safely removed
					-- prevents infinite loop
					table.remove(states, 1);
				end;
			end;
			tkn_i += 1;
		end;
	end;
	if as_bool then
		return true;
	end;
	local match_start_ran = false;
	local span = table.create(token.group_n);
	span[0], span.n = { start_i, str_i }, token.group_n;
	for _, v in ipairs(states) do
		if v[1] == "matchStart" and not match_start_ran then
			span[0][1], match_start_ran = v[2], true;
		elseif v[1] == "group" and v[5] and not span[v[5]] then
			span[v[5]] = { v[3], v[4] };
		end;
	end;
	return span;
end;