版权声明:本文为博主原创文章,遵循 CC 4.0 BY-SA版权协议,转载请附上原文出处链接和本声明。结巴文本分词--文本序列化处理
import jieba from sklearn.model_selection import train_test_split from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences 首先对语句进行分析: x_text = [' '.join(jieba.cut(x)) for x in data] x_text = [ '整个 感觉 除了 送货 师傅 新手 来晚 了外,其他 都很好' , '京东 自营 很 不错 ,' , '外观 手感 使用 都 不错 , 好!' ] tokenizer = Tokenizer(filters='!"#$%&()*+,-./:;?@[\\]^_`{|}~\t\n',lower=True,split=" ") tokenizer.fit_on_texts(x_text) vocab = tokenizer.word_index x_train, x_test, y_train, y_test = train_test_split(x_text, y, test_size=0.2, random_state=2017) x_train_word_ids = tokenizer.texts_to_sequences(x_train) x_test_word_ids = tokenizer.texts_to_sequences(x_test) x_train_padded_seqs = pad_sequences(x_train_word_ids, maxlen=64) x_test_padded_seqs = pad_sequences(x_test_word_ids, maxlen=64) # MLP-onehot x_train_matrix = tokenizer.sequences_to_matrix(x_train_word_ids, mode='binary') x_test_matrix = tokenizer.sequences_to_