private SparseVectorN extractWordGrams(StringTokenizer tokenizer)
    {
        SparseVectorN x = new SparseVectorN();

        assert (ngrams > 0);

        String t = null;
        String l = null;
        String ll = null;
        String lll;

        for (;tokenizer.hasMoreTokens();)
        {

            // Circular
            lll = ll;
            ll = l;
            l = t;
            t = tokenizer.nextToken().toLowerCase().replaceAll("\"", "").replaceAll("'", "").replaceAll("`", "").replaceAll(",", "");
            if (t.isEmpty())
            {
                continue;
            }
            // unigram
            x.add(toFeature(t));

            // bigram?
            if (ngrams > 1 && l != null)
            {
                // trigram?
                x.add(toFeature(l, t));
                if (ngrams > 2 && ll != null)
                {
                    x.add(toFeature(ll, l, t));
                }
                // 4-gram?
                if (ngrams > 3 && lll != null)
                {
                    x.add(toFeature(lll, ll, l, t));
                }
            }
        }
        return x;
    }