ttezel/bayes — pull request #4 by andrepcg (2014-10-05 14:58:17+00:00)
Title: Support for different tokenize functions

The user can now provide a different tokenize function. Eg: tokenize and
remove stopwords

var normalize = function normalize(word) {
return word.toLowerCase().replace(/['"”“\]\[<>#$%&’‘\+\*]/g,"").trim();
};

var stopWords = ["the"];

var tokenize = function (text) {
words = text.split(/[\s\-_():.!\?,;\n]+/);

return words.map(exports.normalize).filter(function(word) {
return word.length > 2 && (!~stopWords.indexOf(word)) && !word.match(/\d/g);
});
}

var classifier = bayes({tokenizer: tokenize})