File size: 2,472 Bytes
e350168 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
# 导入所需的库
import json
import paddlenlp
import gensim
import sklearn
from collections import Counter
from gensim import corpora, models, similarities
import numpy as np
import matplotlib.pyplot as plt
def build_corpus(sentences):
# 使用paddlenlp提供的预训练词典
vocab = paddlenlp.transformers.BertTokenizer.from_pretrained('bert-base-chinese').vocab
# 创建分词器
tokenizer = paddlenlp.data.JiebaTokenizer(vocab)
# 对每个句子进行分词，并去除停用词，得到一个二维列表
stopwords = [""]
words_list = []
for sentence in sentences:
words = [word for word in tokenizer.cut(sentence) if word not in stopwords]
words_list.append(words)
# print(words_list)
# 将二维列表转换为一维列表
words = [word for sentence in words_list for word in sentence]
dictionary = corpora.Dictionary(words_list)
corpus = [dictionary.doc2bow(text) for text in words_list]
return corpus,dictionary,words_list
def lda(words_list,sentences,corpus,dictionary,num):
lda = gensim.models.ldamodel.LdaModel(corpus=corpus,id2word=dictionary, num_topics=num)
topics = lda.print_topics(num_topics=num, num_words=10)
# 根据关键词的匹配度，选择最能代表每个主题的句子，作为中心句
central_sentences = []
for topic in topics:
topic_id, topic_words = topic
topic_words = [word.split("*")[1].strip('"') for word in topic_words.split("+")]
max_score = 0
candidates = [] # 存储候选中心句
for sentence, words in zip(sentences, words_list):
score = 0
for word in words:
if word in topic_words:
score += 1
if score > max_score:
max_score = score
candidates = [sentence] # 如果找到更高的匹配度，更新候选列表
elif score == max_score:
candidates.append(sentence) # 如果匹配度相同，添加到候选列表
for candidate in candidates: # 遍历候选列表
if candidate not in central_sentences: # 检查是否已经存在相同的句子
central_sentence = candidate # 如果不存在，选择为中心句
central_sentences.append(central_sentence)
break # 跳出循环
return central_sentences
def abstruct_main(sentences,num):
corpus,dictionary,words_list = build_corpus(sentences)
central_sentences= lda(words_list, sentences, corpus, dictionary,num)
return central_sentences
|