public List<Token> tokenize(final InputStream stream) throws IOException {

		LineColumnReader reader = new LineColumnReader(new InputStreamReader(stream));
		
		List<Token> tokens = new LinkedList<Token>();
		try {
			
			while (reader.ready()) {
				int c = reader.read();
				
				if (Character.isWhitespace(c)) {
					whitespaceReader.readValue(reader, c);
				} else if (Character.isLetter(c)) {
					tokens.add(identifierReader.getToken(reader, c));
				} else if (Character.isDigit(c)) {
					tokens.add(numberReader.getToken(reader, c));
				} else if (c == '"') {
					tokens.add(stringReader.getToken(reader, c));
				} else {
					Token symbol = symbolReader.getToken(reader, c);
					if (symbol != null) {
						tokens.add(symbol);
					}
				}
			}
			
			return tokens;
		
		} catch (IOException e) {
			e.printStackTrace();
			throw e;
		} finally {
			reader.close();
		}
	}