protected void tokenize() throws IOException {
		while (!hasQuit()) {
			final Token token = tokenizer.nextToken();

			if (token == null) {
				// End of the input. We enqueue everything which is still in the
				// buffer, then call it a day.
				enqueueAll();
				break;
			}

			if (token.hasTag(AreaTag.INDICATOR_AREA)
					&& token.getText().equals("-")) {
				// Found a continuing line. We take care of this in a separate
				// method.
				handleContinuation(token);
				continue;
			}

			if ((token.hasTag(AreaTag.INDICATOR_AREA) && ProgramAreaTokenizer
					.indicatesComment(token.getText().charAt(0)))
					|| token.hasTag(AreaTag.COMMENT)
					|| token.hasTag(AreaTag.COMPILER_DIRECTIVE)) {
				// Possible intervening comment line. Buffer all up to next
				// line.
				buffer(token);
				bufferUpToNextLine();
				continue;
			}

			if (token.hasTag(AreaTag.END_OF_LINE)) {
				// We can only continue the previous line. So if we start
				// processing a new line the current line becomes the line to be
				// continued, and we can enqueue the lines before that one.
				enqueueUpToLine(token.getStart().getLinenumber());
			}

			// Buffer the new token.
			buffer(token);
		}
	}