private static String tokenizeAndCollect(String html) {
        try {
            ByteArrayOutputStream buffer = new ByteArrayOutputStream();
            PrintStream out = new PrintStream(buffer);
            Reader in = new StringReader(html);
            HtmlStreamTokenizer tok = new HtmlStreamTokenizer(in);
            HtmlTag tag = new HtmlTag();
            while (tok.nextToken() != HtmlStreamTokenizer.TT_EOF)
            {
                StringBuffer buf = tok.getWhiteSpace();
                if (buf.length() > 0)
                    out.print(buf.toString());

                if (tok.getTokenType() == HtmlStreamTokenizer.TT_TAG)
                {
                    try
                    {
                        tok.parseTag(tok.getStringValue(), tag);
                        if (tag.getTagType() == HtmlTag.T_UNKNOWN)
                            throw new HtmlException("unkown tag");
                        if (tag.isEmpty())
                            out.print("EMPTY ");
                        out.print(tag.toString());
                    }
                    catch (HtmlException e)
                    {
                        // invalid tag, spit it out
                        out.print("\ninvalid: ");
                        out.print("<" + tok.getStringValue() + ">");
                        out.print("\n");
                    }
                }
                else
                {
                    buf = tok.getStringValue();
                    HtmlStreamTokenizer.unescape(buf);
                    if (buf.length() > 0)
                        out.print(buf.toString());
                }
            }
            return buffer.toString();
        }
        catch (IOException e) {
            throw new RuntimeException(e);
        }
    }