|
import json |
|
import jieba |
|
import re |
|
import requests |
|
import backoff |
|
import time |
|
|
|
|
|
@backoff.on_exception(backoff.expo, requests.exceptions.RequestException) |
|
def post_url(url, headers, payload): |
|
time.sleep(1) |
|
response = requests.request("POST", url, headers=headers, data=payload) |
|
return response |
|
|
|
|
|
def seg(text): |
|
text = text.replace('\n', " ") |
|
sentences = re.split(r'(?<=[。！？.!?: ])\s*', text) |
|
sentences = [string for string in sentences if string != ''] |
|
return sentences |
|
|
|
|
|
def clean_text(text): |
|
text = text.replace('\n', "") |
|
text = re.sub(r"-", " ", text) |
|
text = re.sub(r"\d+/\d+/\d+", "", text) |
|
text = re.sub(r"[0-2]?[0-9]:[0-6][0-9]", "", text) |
|
text = re.sub( |
|
r"/[a-zA-Z]*[:\//\]*[A-Za-z0-9\-_]+\.+[A-Za-z0-9\.\/%&=\?\-_]+/i", "", text) |
|
pure_text = '' |
|
for letter in text: |
|
if letter.isalpha() or letter == ' ': |
|
pure_text += letter |
|
|
|
text = ' '.join(word for word in pure_text.split() if len(word) > 1) |
|
return text |
|
|
|
|
|
def article_to_group(groups, topics): |
|
para = {} |
|
for i in groups: |
|
if not i[1] in para: |
|
para[i[1]] = i[0] |
|
else: |
|
para[i[1]] = para[i[1]] + i[0] |
|
return para |
|
|
|
|
|
def generation(para, max_length): |
|
API_KEY = "IZt1uK9PAI0LiqleqT0cE30b" |
|
SECRET_KEY = "Xv5kHB8eyhNuI1B1G7fRgm2SIPdlxGxs" |
|
|
|
def get_access_token(): |
|
|
|
url = "https://aip.baidubce.com/oauth/2.0/token" |
|
params = {"grant_type": "client_credentials", |
|
"client_id": API_KEY, "client_secret": SECRET_KEY} |
|
return str(requests.post(url, params=params).json().get("access_token")) |
|
|
|
url = "https://aip.baidubce.com/rpc/2.0/nlp/v1/news_summary?charset=UTF-8&access_token=" + get_access_token() |
|
topic = {} |
|
Ai_abstract = [] |
|
for i, (j, k) in enumerate(para.items()): |
|
input_text = k |
|
|
|
payload = json.dumps({ |
|
"content": k, |
|
"max_summary_len": max_length |
|
}) |
|
headers = { |
|
'Content-Type': 'application/json', |
|
'Accept': 'application/json' |
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
topic[j] = (j, k) |
|
Ai_abstract.append(j) |
|
return topic,Ai_abstract |
|
def formate_text(title_dict,outline_list): |
|
formated = [] |
|
for each in outline_list: |
|
if(each not in title_dict.keys()): |
|
formated.append(f"# {each}") |
|
if(each in title_dict.keys()): |
|
formated.append(f"## {each}") |
|
formated.append(title_dict[each][1]) |
|
return formated |
