protected void tokenize() throws IOException {
		while (!hasQuit()) {
			final Token token = tokenizer.nextToken();

			if (token == null) {
				break;

			} else if (token.hasTag(AreaTag.PROGRAM_TEXT_AREA)
					&& !token.hasTag(AreaTag.COMMENT)
					&& !token.hasTag(AreaTag.COMPILER_DIRECTIVE)) {
				separate(token);

			} else {
				enqueue(token);
			}
		}
	}