public Token[] tokenize(String input) 
    {
        List tokens = new ArrayList();
        int cursor = 0;
        while (cursor<input.length()) {
	    char ch = input.charAt(cursor);
	    if (Character.isWhitespace(ch)) {
                cursor++;
	    } else if (Character.isLetter(ch)) {
                StringBuffer buf = new StringBuffer("");
                while (cursor<input.length() && Character.isLetter(input.charAt(cursor))) {
                    buf.append(input.charAt(cursor));
                    cursor++;
                }
                tokens.add(internSomething(buf.toString()));
	    } else if (Character.isDigit(ch)) {
                StringBuffer buf = new StringBuffer("");
                while (cursor<input.length() && Character.isDigit(input.charAt(cursor))) {
                    buf.append(input.charAt(cursor));
                    cursor++;
                }
                tokens.add(internSomething(buf.toString()));
	    } else {
                if (!ignorePunctuation) {
                    StringBuffer buf = new StringBuffer("");
                    buf.append(ch);
                    String str = buf.toString();
                    tokens.add(internSomething(str));
                }
                cursor++;
	    }
        }
        return (Token[]) tokens.toArray(new BasicToken[tokens.size()]);
    }