def __init__(
        self,
        width=800,
        height=400,
        renderer=constants.CANVAS_RENDERER,
        page_title=constants.PAGE_TITLE,
        extra_html_text_label=None,
        is_animation=True,
    ):
        """

        :param width:
            画布宽度，默认为 800（px）
        :param height:
            画布高度，默认为 400（px）
        :param renderer:
            指定使用渲染方式，有 'svg' 和 'canvas' 可选，默认为 'canvas'。
            3D 图仅能使用 'canvas'。
        :param page_title:
            指定生成的 html 文件中 <title> 标签的值。默认为 'Echarts'
        :param extra_html_text_label:
            额外的 HTML 文本标签，(<p> 标签)。类型为 list，list[0] 为文本内容，
            list[1] 为字体风格样式（选填）。如 ["this is a p label", "color:red"]
        :param is_animation:
            是否开启动画，默认为 True。V0.5.9+
        """
        self._option = {}
        self._js_dependencies = set()
        self._chart_id = uuid.uuid4().hex
        self.width, self.height = width, height
        self.renderer = renderer
        self._page_title = page_title
        self._js_dependencies = {"echarts"}
        self.event_handlers = {}
        self.theme = None
        self.use_theme(CURRENT_CONFIG.theme)
        self.extra_html_text_label = extra_html_text_label
        self.is_animation = is_animation

def __init__ ( self , width = 800 , height = 400 , renderer = constants . CANVAS_RENDERER , page_title = constants . PAGE_TITLE , extra_html_text_label = None , is_animation = True , ) : self . _option = { } self . _js_dependencies = set ( ) self . _chart_id = uuid . uuid4 ( ) . hex self . width , self . height = width , height self . renderer = renderer self . _page_title = page_title self . _js_dependencies = { "echarts" } self . event_handlers = { } self . theme = None self . use_theme ( CURRENT_CONFIG . theme ) self . extra_html_text_label = extra_html_text_label self . is_animation = is_animation


def print_echarts_options(self):
        """
        打印输出图形所有配置项
        """
        snippet = TRANSLATOR.translate(self.options)
        print(snippet.as_snippet())

def print_echarts_options ( self ) : snippet = TRANSLATOR . translate ( self . options ) print ( snippet . as_snippet ( ) )


def get_echarts_options(self):
        """
        返回图形所有配置项
        """
        snippet = TRANSLATOR.translate(self.options)
        return snippet.as_snippet()

def get_echarts_options ( self ) : snippet = TRANSLATOR . translate ( self . options ) return snippet . as_snippet ( )


def show_config(self):
        """
        打印输出图形所有配置项
        """
        deprecated_tpl = "The {} is deprecated, please use {} instead!"
        warnings.warn(
            deprecated_tpl.format("show_config", "print_echarts_options"),
            DeprecationWarning,
        )
        self.print_echarts_options()


def render_embed(self):
        """
        渲染图表的所有配置项，为 web pages 服务，不过需先提供所需要的js 依赖文件
        """
        env = engine.create_default_environment(constants.DEFAULT_HTML)
        html = env.render_container_and_echarts_code(self)
        return Markup(html)

def render_embed ( self ) : env = engine . create_default_environment ( constants . DEFAULT_HTML ) html = env . render_container_and_echarts_code ( self ) return Markup ( html )


声明所有的 js 文件路径

声明所有的 js 文件路径

声明所有的 js 文件路径

def get_js_dependencies(self):
        """
        声明所有的 js 文件路径
        """
        return CURRENT_CONFIG.produce_html_script_list(self._js_dependencies)

def get_js_dependencies ( self ) : return CURRENT_CONFIG . produce_html_script_list ( self . _js_dependencies )


转换数据序列，将带字典和元组类型的序列转换为 k_lst,v_lst 两个列表

转换数据序列，将带字典和元组类型的序列转换为

def cast(seq):
        """
        转换数据序列，将带字典和元组类型的序列转换为 k_lst,v_lst 两个列表

        元组列表
            [(A1, B1), (A2, B2), ...] -->
                k_lst[ A[i1, i2...] ], v_lst[ B[i1, i2...] ]
        字典列表
            [{A1: B1}, {A2: B2}, ...] -->
                k_lst[ A[i1, i2...] ], v_lst[ B[i1, i2...] ]
        字典
            {A1: B1, A2: B2, ...} -- >
                k_lst[ A[i1, i2...] ], v_lst[ B[i1, i2...] ]

        :param seq:
            待转换的序列
        :return:
        """
        k_lst, v_lst = [], []
        if isinstance(seq, list):
            for s in seq:
                if isinstance(s, tuple):
                    _attr, _value = s
                    k_lst.append(_attr)
                    v_lst.append(_value)
                elif isinstance(s, dict):
                    for k, v in s.items():
                        k_lst.append(k)
                        v_lst.append(v)
        elif isinstance(seq, dict):
            for key in sorted(list(seq.keys())):
                k_lst.append(key)
                v_lst.append(seq[key])
        return k_lst, v_lst

def cast ( seq ) : k_lst , v_lst = [ ] , [ ] if isinstance ( seq , list ) : for s in seq : if isinstance ( s , tuple ) : _attr , _value = s k_lst . append ( _attr ) v_lst . append ( _value ) elif isinstance ( s , dict ) : for k , v in s . items ( ) : k_lst . append ( k ) v_lst . append ( v ) elif isinstance ( seq , dict ) : for key in sorted ( list ( seq . keys ( ) ) ) : k_lst . append ( key ) v_lst . append ( seq [ key ] ) return k_lst , v_lst


渲染配置项并将图形显示在 notebook 中

渲染配置项并将图形显示在 notebook 中

def _repr_html_(self):
        """
        渲染配置项并将图形显示在 notebook 中

        chart/page => charts
        chart.js_dependencies => require_config => config_items, libraries
        :return A unicode string.
        """
        if CURRENT_CONFIG.jupyter_presentation == constants.DEFAULT_HTML:
            require_config = CURRENT_CONFIG.produce_require_configuration(
                self.js_dependencies
            )
            config_items = require_config["config_items"]
            libraries = require_config["libraries"]
            env = engine.create_default_environment(constants.DEFAULT_HTML)
            return env.render_chart_to_notebook(
                charts=(self,), config_items=config_items, libraries=libraries
            )

        elif CURRENT_CONFIG.jupyter_presentation == constants.NTERACT:
            env = engine.create_default_environment(constants.DEFAULT_HTML)
            return env.render_chart_to_notebook(
                chart=self, template_name="nteract.html"
            )

        else:
            return None

def _repr_html_ ( self ) : if CURRENT_CONFIG . jupyter_presentation == constants . DEFAULT_HTML : require_config = CURRENT_CONFIG . produce_require_configuration ( self . js_dependencies ) config_items = require_config [ "config_items" ] libraries = require_config [ "libraries" ] env = engine . create_default_environment ( constants . DEFAULT_HTML ) return env . render_chart_to_notebook ( charts = ( self , ) , config_items = config_items , libraries = libraries ) elif CURRENT_CONFIG . jupyter_presentation == constants . NTERACT : env = engine . create_default_environment ( constants . DEFAULT_HTML ) return env . render_chart_to_notebook ( chart = self , template_name = "nteract.html" ) else : return None


This is an internal function to serve _repr_jpeg_ _repr_png_ and _repr_svg_ .

def _render_as_image(self, file_type):
        """
        This is an internal function to serve _repr_jpeg_,
        _repr_png_ and _repr_svg_.

        :param file_type: the parameter is mostly image file types.
        """
        if CURRENT_CONFIG.jupyter_presentation != file_type:
            return None

        if self.renderer == constants.SVG_RENDERER:
            if file_type != constants.SVG:
                raise exceptions.InvalidConfiguration(
                    "svg renderer produces only svg image."
                )

        elif file_type not in [constants.JPEG, constants.PNG]:
            # CANVAS_RENDERER here
            raise exceptions.InvalidConfiguration(
                "svg output requires svg renderer."
            )

        env = engine.create_default_environment(file_type)
        tmp_file_handle, tmp_file_path = mkstemp(suffix="." + file_type)
        content = env.render_chart_to_file(
            chart=self, path=tmp_file_path, verbose=False
        )
        os.close(tmp_file_handle)
        return content

def
_render_as_image
(
self
,
file_type
)
:
if
CURRENT_CONFIG
.
jupyter_presentation
!=
file_type
:
return
if
self
.
renderer
==
constants
.
SVG_RENDERER
:
if
file_type
!=
constants
.
SVG
:
raise
exceptions
.
InvalidConfiguration
(
"svg renderer produces only svg image."
)
elif
file_type
not
in
[
constants
.
JPEG
,
constants
.
PNG
]
:
# CANVAS_RENDERER here
raise
exceptions
.
InvalidConfiguration
(
"svg output requires svg renderer."
)
env
=
engine
.
create_default_environment
(
file_type
)
tmp_file_handle
,
tmp_file_path
=
mkstemp
(
suffix
=
"."
+
file_type
)
content
=
env
.
render_chart_to_file
(
chart
=
self
,
path
=
tmp_file_path
,
verbose
=
False
)
os
.
close
(
tmp_file_handle
)
return
content


def copy(self):
        """Return a copy of this Request"""
        return self.replace()

def copy ( self ) : return self . replace ( )


Create a new Request with the same attributes except for those given new values .

def replace(self, *args, **kwargs):
        """Create a new Request with the same attributes except for those
        given new values.
        """
        for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta',
                  'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
            kwargs.setdefault(x, getattr(self, x))
        cls = kwargs.pop('cls', self.__class__)
        return cls(*args, **kwargs)

def replace ( self , * args , * * kwargs ) : for x in [ 'url' , 'method' , 'headers' , 'body' , 'cookies' , 'meta' , 'encoding' , 'priority' , 'dont_filter' , 'callback' , 'errback' ] : kwargs . setdefault ( x , getattr ( self , x ) ) cls = kwargs . pop ( 'cls' , self . __class__ ) return cls ( * args , * * kwargs )


def copy(self):
        """Return a copy of this Response"""
        return self.replace()


Create a new Response with the same attributes except for those given new values .

def replace(self, *args, **kwargs):
        """Create a new Response with the same attributes except for those
        given new values.
        """
        for x in ['url', 'status', 'headers', 'body', 'request', 'flags']:
            kwargs.setdefault(x, getattr(self, x))
        cls = kwargs.pop('cls', self.__class__)
        return cls(*args, **kwargs)

def replace ( self , * args , * * kwargs ) : for x in [ 'url' , 'status' , 'headers' , 'body' , 'request' , 'flags' ] : kwargs . setdefault ( x , getattr ( self , x ) ) cls = kwargs . pop ( 'cls' , self . __class__ ) return cls ( * args , * * kwargs )


Join this Response s url with a possible relative url to form an absolute interpretation of the latter .

def urljoin(self, url):
        """Join this Response's url with a possible relative url to form an
        absolute interpretation of the latter."""
        return urljoin(self.url, url)

def urljoin ( self , url ) : return urljoin ( self . url , url )


For subclasses of TextResponse this will return the body as text ( unicode object in Python 2 and str in Python 3 )

def text(self):
        """For subclasses of TextResponse, this will return the body
        as text (unicode object in Python 2 and str in Python 3)
        """
        raise AttributeError("Response content isn't text")

def
text
(
self
)
:
raise
AttributeError
(
"Response content isn't text"
)


Shortcut method implemented only by responses whose content is text ( subclasses of TextResponse ) .

def css(self, *a, **kw):
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

def
css
(
self
,
*
a
,
*
*
kw
)
:
raise
NotSupported
(
"Response content isn't text"
)


def xpath(self, *a, **kw):
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

def
xpath
(
self
,
*
a
,
*
*
kw
)
:
raise
NotSupported
(
"Response content isn't text"
)


Return a : class : ~ . Request instance to follow a link url . It accepts the same arguments as Request . __init__ method but url can be a relative URL or a scrapy . link . Link object not only an absolute URL . : class : ~ . TextResponse provides a : meth : ~ . TextResponse . follow method which supports selectors in addition to absolute / relative URLs and Link objects .

def follow(self, url, callback=None, method='GET', headers=None, body=None,
               cookies=None, meta=None, encoding='utf-8', priority=0,
               dont_filter=False, errback=None):
        # type: (...) -> Request
        """
        Return a :class:`~.Request` instance to follow a link ``url``.
        It accepts the same arguments as ``Request.__init__`` method,
        but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
        not only an absolute URL.

        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
        method which supports selectors in addition to absolute/relative URLs
        and Link objects.
        """
        if isinstance(url, Link):
            url = url.url
        url = self.urljoin(url)
        return Request(url, callback,
                       method=method,
                       headers=headers,
                       body=body,
                       cookies=cookies,
                       meta=meta,
                       encoding=encoding,
                       priority=priority,
                       dont_filter=dont_filter,
                       errback=errback)

def
follow
(
self
,
url
,
callback
=
,
method
=
'GET'
,
headers
=
,
body
=
,
cookies
=
,
meta
=
,
encoding
=
'utf-8'
,
priority
=
0
,
dont_filter
=
False
,
errback
=
)
:
# type: (...) -> Request
if
isinstance
(
url
,
Link
)
:
url
=
url
.
url
url
=
self
.
urljoin
(
url
)
return
Request
(
url
,
callback
,
method
=
method
,
headers
=
headers
,
body
=
body
,
cookies
=
cookies
,
meta
=
meta
,
encoding
=
encoding
,
priority
=
priority
,
dont_filter
=
dont_filter
,
errback
=
errback
)


def __init__(self, phone):
        """
        :param phone:adb 操作手机所需要的端口信息
        """
        # 一下写法不理解参考https://stackoverflow.com/questions/11179008/python-inheritance-typeerror-object-init-takes-no-parameters
        PhoneControl.__init__(self,phone=phone)


点击指定words所在的区域 : param words : 目标文字 : return : 实际点击的位置

def click_by_words(self,words,tap=True):
        """
        点击指定words所在的区域
        :param words:目标文字
        :return:实际点击的位置
        """
        # 截图
        pic_name = self.get_screen_cap()
        # COR识别ui_words
        ui_words = VC.ocr(pic_name ,location=True)
        # 根据words找最相似
        prob, loc_words = VC.find_position(ui_words, words)
        # 执行点击
        loc = loc_words['location']
        pos = [loc['left'],loc['top'],loc['left']+loc['width'],loc['top']+loc['height']]
        if tap is True:
            self.input_tap(pos)
        return pos

def
click_by_words
(
self
,
words
,
tap
=
True
)
:
# 截图
pic_name
=
self
.
get_screen_cap
(
)
# COR识别ui_words
ui_words
=
VC
.
ocr
(
pic_name
,
location
=
True
)
# 根据words找最相似
prob
,
loc_words
=
VC
.
find_position
(
ui_words
,
words
)
# 执行点击
loc
=
loc_words
[
'location'
]
pos
=
[
loc
[
'left'
]
,
loc
[
'top'
]
,
loc
[
'left'
]
+
loc
[
'width'
]
,
loc
[
'top'
]
+
loc
[
'height'
]
]
if
tap
is
True
:
self
.
input_tap
(
pos
)
return
pos


根据区域位置点击屏幕 同时支持 点位置 : param pos : 区域左上角和右下角的坐标 : return :

def click_by_loc(self,pos):
        """
        根据区域位置点击屏幕 同时支持 点位置
        :param pos:区域左上角和右下角的坐标
        :return:
        """
        acture_pos = self.input_tap(pos)
        return acture_pos

def click_by_loc ( self , pos ) : acture_pos = self . input_tap ( pos ) return acture_pos


根据给定的关键字返回它们的位置列表 并且给出相似度 大于0 . 9基本可信 : param keys : ( key1 key2 ... ) : param crop : 聚焦裁剪区域 该tuple中信息为 ( left upper right lower ) 左上角为原点 : return : { key1 : { pos : [ x1 y1 x2 y2 ] prob : 0 . 998 } key1 : { pos : [ x1 y1 x2 y2 ] prob : 0 . 998 } ... }

def x_ray(self, keys, crop=None):
        """
        根据给定的关键字返回它们的位置列表 并且给出相似度 大于0.9基本可信
        :param keys:('key1','key2',...)
        :param crop:聚焦裁剪区域 该tuple中信息为(left, upper, right, lower) 左上角为原点
        :return:{'key1':{'pos':[x1,y1,x2,y2],'prob':0.998},'key1':{'pos':[x1,y1,x2,y2],'prob':0.998},...}
        """
        x_ray_data = {}
        # 截图
        pic_name = self.get_screen_cap()
        # 获取图片的ui_words
        ui_words = VC.ocr(pic_name ,location=True, crop=crop)
        for key in keys:
            x_ray_data[key] = {}
            # 找到最相似words
            prob, loc_words = VC.find_position(ui_words, key)
            loc = loc_words['location']
            # 长宽转化为右下角坐标
            pos = [loc['left'],loc['top'],loc['left']+loc['width'],loc['top']+loc['height']]
            x_ray_data[key]['pos'] = pos
            x_ray_data[key]['prob'] = prob
        return x_ray_data

def
x_ray
(
self
,
keys
,
crop
=
)
:
x_ray_data
=
{
}
# 截图
pic_name
=
self
.
get_screen_cap
(
)
# 获取图片的ui_words
ui_words
=
VC
.
ocr
(
pic_name
,
location
=
True
,
crop
=
crop
)
for
key
in
keys
:
x_ray_data
[
key
]
=
{
}
# 找到最相似words
prob
,
loc_words
=
VC
.
find_position
(
ui_words
,
key
)
loc
=
loc_words
[
'location'
]
# 长宽转化为右下角坐标
pos
=
[
loc
[
'left'
]
,
loc
[
'top'
]
,
loc
[
'left'
]
+
loc
[
'width'
]
,
loc
[
'top'
]
+
loc
[
'height'
]
]
x_ray_data
[
key
]
[
'pos'
]
=
pos
x_ray_data
[
key
]
[
'prob'
]
=
prob
return
x_ray_data


根据截屏文字信息获得词典用于接下来创建one_hot向量 : param ui_words : 来自ocr结果 : return : 字典 暂时为未添加停用词

def ui_words2vocb(ui_words):
        """
        根据截屏文字信息获得词典用于接下来创建one_hot向量
        :param ui_words:来自ocr结果
        :return:字典 暂时为未添加停用词
        """
        vocab = ''
        for words in ui_words:
            vocab+=words['words']
        vocab = set(vocab)
        return vocab

def ui_words2vocb ( ui_words ) : vocab = '' for words in ui_words : vocab += words [ 'words' ] vocab = set ( vocab ) return vocab


第一个返回值为相似度，第二个返回值为对应的ocr信息，包含文本和位置信息

def find_position(ui_words, dest_words):
        """
        根据cor识别的ui_words，找到与指定目标dest_words最相似的位置
        :param ui_words:
        :param dest_words:
        :return:第一个返回值为相似度，第二个返回值为对应的ocr信息，包含文本和位置信息
        0.998, {'location': {'width': 119, 'top': 1863, 'left': 345, 'height': 40}, 'words': '通讯录'}
        """
        vocab = VC.ui_words2vocb(ui_words)
        dictionary = list(vocab)
        vec_len = len(dictionary)
        keys = range(vec_len)
        dictionary = dict(zip(dictionary, keys))
        # 计算当ui界面每一行words的one-hot向量
        ui_words_vec = []
        for words in ui_words:
            words_vec = np.zeros(vec_len)
            for word in words['words']:
                words_vec[dictionary[word]]+=1
            ui_words_vec.append(words_vec)
        ui_words_vec = np.array(ui_words_vec)

        # 根据词典计算机目标words的one-hot词向量
        dest_words_vec = np.zeros(vec_len)
        for word in dest_words:
            if word not in dictionary:continue
            dest_words_vec[dictionary[word]]+=1

        #计算余弦夹角
        cos_sim = []
        for vec in ui_words_vec:
            cs = VC.cos_sim(dest_words_vec,vec)
            cos_sim.append(cs)

        #寻找最可能点击位置
        max_index = cos_sim.index(max(cos_sim))
        return cos_sim[max_index], ui_words[max_index]

def
find_position
(
ui_words
,
dest_words
)
:
vocab
=
VC
.
ui_words2vocb
(
ui_words
)
dictionary
=
list
(
vocab
)
vec_len
=
len
(
dictionary
)
keys
=
range
(
vec_len
)
dictionary
=
dict
(
zip
(
dictionary
,
keys
)
)
# 计算当ui界面每一行words的one-hot向量
ui_words_vec
=
[
]
for
words
in
ui_words
:
words_vec
=
np
.
zeros
(
vec_len
)
for
word
in
words
[
'words'
]
:
words_vec
[
dictionary
[
word
]
]
+=
1
ui_words_vec
.
append
(
words_vec
)
ui_words_vec
=
np
.
array
(
ui_words_vec
)
# 根据词典计算机目标words的one-hot词向量
dest_words_vec
=
np
.
zeros
(
vec_len
)
for
word
in
dest_words
:
if
word
not
in
dictionary
:
continue
dest_words_vec
[
dictionary
[
word
]
]
+=
1
#计算余弦夹角
cos_sim
=
[
]
for
vec
in
ui_words_vec
:
cs
=
VC
.
cos_sim
(
dest_words_vec
,
vec
)
cos_sim
.
append
(
cs
)
#寻找最可能点击位置
max_index
=
cos_sim
.
index
(
max
(
cos_sim
)
)
return
cos_sim
[
max_index
]
,
ui_words
[
max_index
]


计算两个向量的余弦相似性，结果越接近1越相似 : param a : : param b : : return :

def cos_sim(a,b):
        """
        计算两个向量的余弦相似性，结果越接近1越相似
        :param a:
        :param b:
        :return:
        """
        cs = np.dot(a, b)/(np.linalg.norm(a)*np.linalg.norm(b))
        return cs

def cos_sim ( a , b ) : cs = np . dot ( a , b ) / ( np . linalg . norm ( a ) * np . linalg . norm ( b ) ) return cs


装饰器负责每次命令前连接手机
    :param func:
    :return:

装饰器负责每次命令前连接手机
    :param func:
    :return:

装饰器负责每次命令前连接手机 : param func : : return :

def connect_phone(func):
    """
    装饰器负责每次命令前连接手机
    :param func:
    :return:
    """
    def wrapper(*args, **kwargs):
        system('adb connect '+args[0].phone)
        return func(*args, **kwargs)
    return wrapper

def
connect_phone
(
func
)
:
def
wrapper
(
*
args
,
*
*
kwargs
)
:
system
(
'adb connect '
+
args
[
0
]
.
phone
)
return
func
(
*
args
,
*
*
kwargs
)
return
wrapper


def __init__(self, phone):
        """
        :param phone:adb 操作手机所需要的端口信息
        """
        self.phone = phone

def __init__ ( self , phone ) : self . phone = phone


def get_phone(self):
        """
        获取实例手机端口信息
        """
        return self.phone

def get_phone ( self ) : return self . phone


def get_screen_cap(self, file_name='screen_cap'):
        """
        获取截图
        """
        system('adb -s '+str(self.phone)+' shell screencap -p /sdcard/'+file_name+'.png')
        system('adb -s '+str(self.phone)+' pull /sdcard/'+file_name+'.png'+' ./'+file_name+'.png')
        return file_name+'.png'

def
get_screen_cap
(
self
,
file_name
=
'screen_cap'
)
:
system
(
'adb -s '
+
str
(
self
.
phone
)
+
' shell screencap -p /sdcard/'
+
file_name
+
'.png'
)
system
(
'adb -s '
+
str
(
self
.
phone
)
+
' pull /sdcard/'
+
file_name
+
'.png'
+
' ./'
+
file_name
+
'.png'
)
return
file_name
+
'.png'


def input_tap(self, pos):
        """
        点击屏幕
        pos为一个区域的左上与右下坐标
        如果事先已经随机化了pos可以只是一个点 随机点击一个位置是为防止被被认为是机器人
        返回实际点击位置
        """
        if len(pos) == 2:_pos = pos
        else : _pos = (random.randint(pos[0],pos[2]),random.randint(pos[1],pos[3]))
        command = r'adb -s '+str(self.phone)+' shell input tap {} {}'.format(_pos[0],_pos[1])
        system(command)
        return _pos

def
input_tap
(
self
,
pos
)
:
if
len
(
pos
)
==
2
:
_pos
=
pos
else
:
_pos
=
(
random
.
randint
(
pos
[
0
]
,
pos
[
2
]
)
,
random
.
randint
(
pos
[
1
]
,
pos
[
3
]
)
)
command
=
r'adb -s '
+
str
(
self
.
phone
)
+
' shell input tap {} {}'
.
format
(
_pos
[
0
]
,
_pos
[
1
]
)
system
(
command
)
return
_pos


从一个位置滑动到另一位置 : param x1 : 起始坐标，屏幕左上角为原点 : param x2 : 终点坐标，屏幕左上角为原点 : return : [ x1 x2 ]

def input_swipe(self, x1, x2):
        """
        从一个位置滑动到另一位置
        :param x1:起始坐标，屏幕左上角为原点
        :param x2:终点坐标，屏幕左上角为原点
        :return:[x1,x2]
        """
        command = r'adb -s '+str(self.phone)+' shell input swipe {} {} {} {}'.format(x1[0],x1[1],x2[0],x2[1])
        system(command)
        return [x1,x2]

def
input_swipe
(
self
,
x1
,
x2
)
:
command
=
r'adb -s '
+
str
(
self
.
phone
)
+
' shell input swipe {} {} {} {}'
.
format
(
x1
[
0
]
,
x1
[
1
]
,
x2
[
0
]
,
x2
[
1
]
)
system
(
command
)
return
[
x1
,
x2
]


拉动屏幕 : param dx : x方向速度 : param dy : y方向速度 : return : [ dx dy ]

def input_roll(self, dx=0, dy=500):
        """
        拉动屏幕
        :param dx:x方向速度
        :param dy:y方向速度
        :return:[dx, dy]
        """
        command = r'adb -s '+str(self.phone)+' shell input roll {} {}'.format(dx,dy)
        system(command)
        return [dx,dy]

def
input_roll
(
self
,
dx
=
0
,
dy
=
500
)
:
command
=
r'adb -s '
+
str
(
self
.
phone
)
+
' shell input roll {} {}'
.
format
(
dx
,
dy
)
system
(
command
)
return
[
dx
,
dy
]


按键事件 比如home menue back volum_up volum_down等等 具体定义在配置文件中 : param event_cmd : 事件ID : return : event_cmd

def input_key_event(self, event_cmd):
        """
        按键事件 比如home menue back volum_up volum_down等等 具体定义在配置文件中
        :param event_cmd:事件ID
        :return:event_cmd
        """
        command = r'adb -s '+str(self.phone)+' shell input keyevent '+event_cmd
        system(command)
        return event_cmd

def
input_key_event
(
self
,
event_cmd
)
:
command
=
r'adb -s '
+
str
(
self
.
phone
)
+
' shell input keyevent '
+
event_cmd
system
(
command
)
return
event_cmd


输入文本信息 可能不支持中文输入 : param text : 待输入的文本信息 : return : 文本信息

def input_text(self, text):
        """
        输入文本信息 可能不支持中文输入
        :param text:待输入的文本信息
        :return:文本信息
        """
        command = r'adb -s '+str(self.phone)+' shell input text {}'.format(text)
        system(command)
        return text

def
input_text
(
self
,
text
)
:
command
=
r'adb -s '
+
str
(
self
.
phone
)
+
' shell input text {}'
.
format
(
text
)
system
(
command
)
return
text


支持中文 需要事先将ADB键盘设置为默认输入法而且打开软键盘 : param text : 待输入的文本信息 : return : 文本信息

def input_chn(self, text):
        """
        支持中文 需要事先将ADB键盘设置为默认输入法而且打开软键盘
        :param text:待输入的文本信息
        :return:文本信息
        """
        # adb -s 127.0.0.1:62001 shell am broadcast -a ADB_INPUT_TEXT --es msg "威武之师"
        # command = r'adb -s '+str(self.phone)+' shell am broadcast -a ADB_INPUT_TEXT --es msg {}'.format(text)
        command = r'adb -s '+str(self.phone)+' shell am broadcast -a ADB_INPUT_TEXT --es msg {}'.format(text)
        system(command)
        return text


def __init__(self, phone_list):
        """
        :param phone_list:
        """
        self.phone_list = phone_list
        self.pcs = []
        for ap in self.phone_list:
            self.pcs.append(PhoneControl(ap))

def __init__ ( self , phone_list ) : self . phone_list = phone_list self . pcs = [ ] for ap in self . phone_list : self . pcs . append ( PhoneControl ( ap ) )


: param operation : PhoneControl实例方法字符串名称 : param args : tuple格式参数 : return :

def operate_all(self, operation, args):
        """
        :param operation: PhoneControl实例方法字符串名称
        :param args:tuple格式参数
        :return:
        """
        _tasks = []
        for pc in self.pcs:
            _tasks.append(Thread(target=pc.__getattribute__(operation), args=args))
        for t in _tasks:
            t.start()
        for t in _tasks:
            t.join()
        return operation

def operate_all ( self , operation , args ) : _tasks = [ ] for pc in self . pcs : _tasks . append ( Thread ( target = pc . __getattribute__ ( operation ) , args = args ) ) for t in _tasks : t . start ( ) for t in _tasks : t . join ( ) return operation


:param filePath: 文件地址
        :return: 读取的图片文件

:param filePath: 文件地址
        :return: 读取的图片文件

: param filePath : 文件地址 : return : 读取的图片文件

def get_file_content(filePath):
        """
        :param filePath: 文件地址
        :return: 读取的图片文件
        """
        with open(filePath, 'rb') as fp:
            return fp.read()

def get_file_content ( filePath ) : with open ( filePath , 'rb' ) as fp : return fp . read ( )


: param pic_file_name : 待压缩图片名称 : param quality : 质量百分数 : param crop : ( left top right bottom ) 裁剪左上角和右下角的绝对坐标 : return : 压缩图片并且返回压缩只有图片的名称

def pre_process_img(pic_file_name, quality=50, crop=None):
        """
        :param pic_file_name:待压缩图片名称
        :param quality:质量百分数
        :param crop:(left, top, right, bottom)裁剪左上角和右下角的绝对坐标
        :return: 压缩图片并且返回压缩只有图片的名称
        """
        im = Image.open(pic_file_name)
        rgb_im = im.convert('RGB')
        if crop:
            rgb_im = rgb_im.crop(crop)
        rgb_im.save(pic_file_name+'.jpg', optimize=True, quality=quality)
        return pic_file_name+'.jpg'

def pre_process_img ( pic_file_name , quality = 50 , crop = None ) : im = Image . open ( pic_file_name ) rgb_im = im . convert ( 'RGB' ) if crop : rgb_im = rgb_im . crop ( crop ) rgb_im . save ( pic_file_name + '.jpg' , optimize = True , quality = quality ) return pic_file_name + '.jpg'


根据包含路径的图片文件名调用API返回cor结果 : param pic_file_name : 根据包含路径的图片文件名 : param location : 是否需要识别位置信息 需要位置信息 True 表示需要 : return : 有位置信息 [ { location : { height : 59 left : 38 top : 827 width : 432 } words : 网陈苏苏 : [ 链接 ] } { location : { height : 56 left : 212 top : 955 width : 206 } words : 服务通知 } ... ] 无位置信息 [ { words : 微信 } { words : 通讯录 } { words : 发现 } { words : 我 } ] ... ]

def ocr(pic_file_name, location=False, quality=50, crop=None):
        """
        根据包含路径的图片文件名调用API返回cor结果
        :param pic_file_name:根据包含路径的图片文件名
        :param location:是否需要识别位置信息 需要位置信息 True 表示需要
        :return:
        有位置信息
        [{'location': {'height': 59,'left': 38,'top': 827,'width': 432},
          'words': '网陈苏苏:[链接]'},
         {'location': {'height': 56,'left': 212,'top': 955,'width': 206},
          'words': '服务通知'},...]
        无位置信息
        [{'words': '微信'},
         {'words': '通讯录'},
         {'words': '发现'},
         {'words': '我'}],...
        ]
        """
        compressed_image = OCR.pre_process_img(pic_file_name,quality=quality,crop=crop)
        image = OCR.get_file_content(compressed_image)
        # 带有位置信息
        if location:
            try:
                result = client.general(image)
            except Exception as e:
                logging.error("请求 带位置信息 OCR失败 请检查网络或API可用次数")
                exit()
            # 计算words裁剪之前的真是坐标
            if crop:
                for words in result['words_result']:
                    words['location']['left'] += crop[0]
                    words['location']['top'] += crop[1]
        # 不带位置信息
        else:
            try:
                result = client.basicGeneral(image)
            except Exception as e:
                logging.error("请求 非位置信息 OCR失败 请检查网络或API可用次数")
                exit()
        if result['words_result_num'] != 0:
            return result['words_result']
        else:
            return OCR_NO_WORDS

def
ocr
(
pic_file_name
,
location
=
False
,
quality
=
50
,
crop
=
)
:
compressed_image
=
OCR
.
pre_process_img
(
pic_file_name
,
quality
=
quality
,
crop
=
crop
)
image
=
OCR
.
get_file_content
(
compressed_image
)
# 带有位置信息
if
location
:
try
:
result
=
client
.
general
(
image
)
except
Exception
as
e
:
logging
.
error
(
"请求 带位置信息 OCR失败 请检查网络或API可用次数")
exit
(
)
# 计算words裁剪之前的真是坐标
if
crop
:
for
words
in
result
[
'words_result'
]
:
words
[
'location'
]
[
'left'
]
+=
crop
[
0
]
words
[
'location'
]
[
'top'
]
+=
crop
[
1
]
# 不带位置信息
else
:
try
:
result
=
client
.
basicGeneral
(
image
)
except
Exception
as
e
:
logging
.
error
(
"请求 非位置信息 OCR失败 请检查网络或API可用次数")
exit
(
)
if
result
[
'words_result_num'
]
!=
0
:
return
result
[
'words_result'
]
else
:
return
OCR_NO_WORDS


: return : 通过多次点击BACK按键回到主界面 之所以不直接点击HOME按键 是需要层层返回微信到主界面

def home(self):
        """
        :return:通过多次点击BACK按键回到主界面 之所以不直接点击HOME按键 是需要层层返回微信到主界面
        """
        for i in range(7):
            self.oap.key(KEY['BACK_KEYEVENT'])
            time.sleep(0.3)
        return KEY['BACK_KEYEVENT']

def home ( self ) : for i in range ( 7 ) : self . oap . key ( KEY [ 'BACK_KEYEVENT' ] ) time . sleep ( 0.3 ) return KEY [ 'BACK_KEYEVENT' ]


:return:从主界面到公众号搜索

:return:从主界面到公众号搜索

: return : 从主界面到公众号搜索

def home_to_gzh_search(self):
        """
        :return:从主界面到公众号搜索
        """
        # 点击微信图标
        self.oap.tap(BTN['EMU_WEIXIN_ICON'])
        time.sleep(0.5)
        # 点击通信录
        self.oap.tap(BTN['TONGXUNLU_BTN'])
        time.sleep(0.5)
        # 点击公众号
        self.oap.tap(BTN['GZH_FOLDER'])
        time.sleep(0.5)
        # 点击搜索
        self.oap.tap(BTN['SEARCH_BTN'])
        time.sleep(1)
        return 0

def
home_to_gzh_search
(
self
)
:
# 点击微信图标
self
.
oap
.
tap
(
BTN
[
'EMU_WEIXIN_ICON'
]
)
time
.
sleep
(
0.5
)
# 点击通信录
self
.
oap
.
tap
(
BTN
[
'TONGXUNLU_BTN'
]
)
time
.
sleep
(
0.5
)
# 点击公众号
self
.
oap
.
tap
(
BTN
[
'GZH_FOLDER'
]
)
time
.
sleep
(
0.5
)
# 点击搜索
self
.
oap
.
tap
(
BTN
[
'SEARCH_BTN'
]
)
time
.
sleep
(
1
)
return
0


:param nickname:待搜索公众号名称
        :return:

:param nickname:待搜索公众号名称
        :return:

def search_gzh(self, nickname):
        """
        :param nickname:待搜索公众号名称
        :return:
        """
        # 输入拼音
        self.oap.text(nickname)
        time.sleep(0.5)
        # 进入账号
        self.oap.tap(BTN['FIRST_GZH_SEARCH_RESULT'])
        time.sleep(0.5)
        #键入主界面
        self.oap.tap(BTN['PROFILE_BTN'])
        time.sleep(0.5)
        # 上拉
        self.oap.roll(0,500)
        time.sleep(0.5)
        return 0

def
search_gzh
(
self
,
nickname
)
:
# 输入拼音
self
.
oap
.
text
(
nickname
)
time
.
sleep
(
0.5
)
# 进入账号
self
.
oap
.
tap
(
BTN
[
'FIRST_GZH_SEARCH_RESULT'
]
)
time
.
sleep
(
0.5
)
#键入主界面
self
.
oap
.
tap
(
BTN
[
'PROFILE_BTN'
]
)
time
.
sleep
(
0.5
)
# 上拉
self
.
oap
.
roll
(
0
,
500
)
time
.
sleep
(
0.5
)
return
0


:return:从公众号主页下拉点击全部消息消息

:return:从公众号主页下拉点击全部消息消息

: return : 从公众号主页下拉点击全部消息消息

def all_message(self):
        """
        :return:从公众号主页下拉点击全部消息消息
        """
        # 全部消息
        all_message_pos = self.vc.click_by_words("全部消息",tap=False)
        self.oap.tap(all_message_pos)
        time.sleep(5)
        self.oap.roll(0,500)
        time.sleep(2)
        return 0

def
all_message
(
self
)
:
# 全部消息
all_message_pos
=
self
.
vc
.
click_by_words
(
"全部消息",tap=Fal
s
e)
self
.
oap
.
tap
(
all_message_pos
)
time
.
sleep
(
5
)
self
.
oap
.
roll
(
0
,
500
)
time
.
sleep
(
2
)
return
0


:return:来到历史列表之后随机点击一篇文章

:return:来到历史列表之后随机点击一篇文章

: return : 来到历史列表之后随机点击一篇文章

def click_a_message(self, args=2):
        """
        :return:来到历史列表之后随机点击一篇文章
        """
        #获取界面文章标题消息
        if args==1:corp = CROP_RANGE['PROFILE_MESSAGE_LIST']
        elif args==2:corp = CROP_RANGE['MESSAGE_LIST']
        ui_words = self.vc.get_ui_words(location=True, crop=corp)
        #随便点一个标题
        random_index = randint(1,len(ui_words))-1
        loc = ui_words[random_index]['location']
        pos = [loc['left'],loc['top'],loc['left']+loc['width'],loc['top']+loc['height']]
        self.oap.tap(pos)
        #等待页面加载完毕
        time.sleep(5)
        self.oap.roll(0,500)
        time.sleep(1)

def click_a_message ( self , args = 2 ) : #获取界面文章标题消息 if args == 1 : corp = CROP_RANGE [ 'PROFILE_MESSAGE_LIST' ] elif args == 2 : corp = CROP_RANGE [ 'MESSAGE_LIST' ] ui_words = self . vc . get_ui_words ( location = True , crop = corp ) #随便点一个标题 random_index = randint ( 1 , len ( ui_words ) ) - 1 loc = ui_words [ random_index ] [ 'location' ] pos = [ loc [ 'left' ] , loc [ 'top' ] , loc [ 'left' ] + loc [ 'width' ] , loc [ 'top' ] + loc [ 'height' ] ] self . oap . tap ( pos ) #等待页面加载完毕 time . sleep ( 5 ) self . oap . roll ( 0 , 500 ) time . sleep ( 1 )


:return:成功打开一篇文章之后 拉到底检查评论信息

:return:成功打开一篇文章之后 拉到底检查评论信息

: return : 成功打开一篇文章之后 拉到底检查评论信息

def check_comments(self):
        """
        :return:成功打开一篇文章之后 拉到底检查评论信息
        """
        # 拉到底
        for i in range(2):
            self.oap.roll(0,500)
            time.sleep(1)
        time.sleep(2)
        # 检查有无评论 有评论 无评论 有广告 三种情况
        ui_words_str = self.vc.get_ui_words(location=False,in_str=True,crop=CROP_RANGE['LEAVE_MSG_BOTTOM'])
        # 如果暂无评论点击了留言按钮
        if UI_WORDS['NO_LEAVING_MSG'] in ui_words_str:
            print('点击了留言信息。。。')
            self.oap.tap(BTN['LEAVE_MSG'])
            time.sleep(1)
            self.oap.key(KEY['BACK_KEYEVENT'])

def
check_comments
(
self
)
:
# 拉到底
for
i
in
range
(
2
)
:
self
.
oap
.
roll
(
0
,
500
)
time
.
sleep
(
1
)
time
.
sleep
(
2
)
# 检查有无评论 有评论 无评论 有广告 三种情况
ui_words_str
=
self
.
vc
.
get_ui_words
(
location
=
False
,
in_str
=
True
,
crop
=
CROP_RANGE
[
'LEAVE_MSG_BOTTOM'
]
)
# 如果暂无评论点击了留言按钮
if
UI_WORDS
[
'NO_LEAVING_MSG'
]
in
ui_words_str
:
print
(
'点击了留言信息。。。')
self
.
oap
.
tap
(
BTN
[
'LEAVE_MSG'
]
)
time
.
sleep
(
1
)
self
.
oap
.
key
(
KEY
[
'BACK_KEYEVENT'
]
)


获取关于一个公众号的全部请求数据 当前程序使用baidu API受到网络和并发限制效果并十分理想 : param nickname : 公众号昵称 : return : 最后成功与否取决在redis中是否找到有有效数据

def get_all_req_data(self, nickname, hand=False):
        """
        获取关于一个公众号的全部请求数据 当前程序使用baidu API受到网络和并发限制效果并十分理想
        :param nickname: 公众号昵称
        :return:最后成功与否取决在redis中是否找到有有效数据
        """
        TidyReqData.flush_data("*.req")
        redis_instance.set('current_nickname',nickname)
        self.home_to_gzh_search()
        self.search_gzh(nickname)
        if hand==False:
            self.all_message()
            self.click_a_message()
            # self.check_comments()
        else:
            input("请一一手动或取参数 回车退出")
        self.home()

def
get_all_req_data
(
self
,
nickname
,
hand
=
False
)
:
TidyReqData
.
flush_data
(
"*.req"
)
redis_instance
.
set
(
'current_nickname'
,
nickname
)
self
.
home_to_gzh_search
(
)
self
.
search_gzh
(
nickname
)
if
hand
==
False
:
self
.
all_message
(
)
self
.
click_a_message
(
)
# self.check_comments()
else
:
input
(
"请一一手动或取参数 回车退出")
self
.
home
(
)


仅获取阅读量和评论的请求数据 : param nickname : 公众号昵称 : return : 最后成功与否取决在redis中是否找到有有效数据

def get_part_req_data(self, nickname):
        """
        仅获取阅读量和评论的请求数据
        :param nickname:公众号昵称
        :return:最后成功与否取决在redis中是否找到有有效数据
        """
        TidyReqData.flush_data()
        redis_instance.set('current_nickname',nickname)
        self.home_to_gzh_search()
        self.search_gzh(nickname)
        self.click_a_message(args=1)
        self.check_comments()
        self.home()

def get_part_req_data ( self , nickname ) : TidyReqData . flush_data ( ) redis_instance . set ( 'current_nickname' , nickname ) self . home_to_gzh_search ( ) self . search_gzh ( nickname ) self . click_a_message ( args = 1 ) self . check_comments ( ) self . home ( )


: param config : 1 只获取历史文章列表 2 获取文章内容并转化为markdown格式 3 在2的基础上获取文章的阅读量、点赞量、评论量等信息 : return : 数据存入mongodb 无返回值

def gzh_crawler(config=2):
    """
    :param config:
    1 只获取历史文章列表
    2 获取文章内容并转化为markdown格式
    3 在2的基础上获取文章的阅读量、点赞量、评论量等信息
    :return: 数据存入mongodb 无返回值
    """
    deepth= config
    yield runner.crawl(ArticleListSpider)
    deepth -= 1
    if deepth>0:
        yield runner.crawl(ArticleSpider)
    deepth -= 1
    if deepth>0:
        yield runner.crawl(ArticleReadDataSpider)
    reactor.stop()

def gzh_crawler ( config = 2 ) : deepth = config yield runner . crawl ( ArticleListSpider ) deepth -= 1 if deepth > 0 : yield runner . crawl ( ArticleSpider ) deepth -= 1 if deepth > 0 : yield runner . crawl ( ArticleReadDataSpider ) reactor . stop ( )


: param config : 同gzh_crawler 默认获取文章的文本数据 : return :

def run_crawl(config=2):
    """
    :param config:同gzh_crawler 默认获取文章的文本数据
    :return:
    """
    gzh_crawler(config)
    reactor.run()
    print('所有爬虫结束工作')
    # 由于twisted reactor无法重新启动 一个公众号的爬取任务结束之后重启脚本
    from tools import restart
    restart()


def _run_crawl(config=2):
    """
    :param config:同gzh_crawler 默认获取文章的文本数据
    :return:
    """
    from multiprocessing import Process
    p = Process(target=_run_crawl,args=(config,))
    p.start()
    p.join()

def _run_crawl ( config = 2 ) : from multiprocessing import Process p = Process ( target = _run_crawl , args = ( config , ) ) p . start ( ) p . join ( )


: return : 从crapy的全局设置脚本settings . py中获取所有的设置文件

def get_global_settings():
    """
    :return:从crapy的全局设置脚本settings.py中获取所有的设置文件
    """
    custom_settings = {}
    keys = [item for item in dir(settings) if not item.startswith("__")]
    for key in keys:
        custom_settings[key] = getattr(settings, key)
    return custom_settings

def get_global_settings ( ) : custom_settings = { } keys = [ item for item in dir ( settings ) if not item . startswith ( "__" ) ] for key in keys : custom_settings [ key ] = getattr ( settings , key ) return custom_settings


: param args : : param kwargs : 实例化爬虫需要调用的函数

def __init__(self, *args, **kwargs):
        """
        :param args:
        :param kwargs:
        实例化爬虫需要调用的函数
        """
        self.current_nickname = ''

def __init__ ( self , * args , * * kwargs ) : self . current_nickname = ''


: return : 重新爬虫的入口函数 否者直接请求start_urls中的各个url 重写之后手动调用Request并指定回调函数例如self . parse

def start_requests(self):
        """
        :return:重新爬虫的入口函数, 否者直接请求start_urls中的各个url
        重写之后手动调用Request并指定回调函数例如self.parse
        """
        yield Request(url='http://www.aii.com',
                      meta={"list_offset":self.list_offset} ,
                      callback=self.parse, dont_filter=True)

def start_requests ( self ) : yield Request ( url = 'http://www.aii.com' , meta = { "list_offset" : self . list_offset } , callback = self . parse , dont_filter = True )


:param response:
        :return:请求完成之后的回调函数

:param response:
        :return:请求完成之后的回调函数

: param response : : return : 请求完成之后的回调函数

def parse(self, response):
        """
        :param response:
        :return:请求完成之后的回调函数
        """
        self.counter += 1
        cmc = response.get_ext_data['can_msg_continue']
        next_offset = response.get_ext_data['next_offset']
        item = LoadMoreItem()
        item['article_list'] = response.get_ext_data['data']
        item['nickname'] = response.get_ext_data['nickname']
        self.current_nickname = response.get_ext_data['nickname']
        gc.report_crawling({'nickname':item['nickname'],
                            'percent':'UNK',
                            'more':cmc,
                            'title':len(item['article_list'])})
        yield item
        if cmc == 1:
            yield Request(url='http://www.aii.com',
                          meta={"list_offset":next_offset} ,
                          callback=self.parse, dont_filter=True)

def parse ( self , response ) : self . counter += 1 cmc = response . get_ext_data [ 'can_msg_continue' ] next_offset = response . get_ext_data [ 'next_offset' ] item = LoadMoreItem ( ) item [ 'article_list' ] = response . get_ext_data [ 'data' ] item [ 'nickname' ] = response . get_ext_data [ 'nickname' ] self . current_nickname = response . get_ext_data [ 'nickname' ] gc . report_crawling ( { 'nickname' : item [ 'nickname' ] , 'percent' : 'UNK' , 'more' : cmc , 'title' : len ( item [ 'article_list' ] ) } ) yield item if cmc == 1 : yield Request ( url = 'http://www.aii.com' , meta = { "list_offset" : next_offset } , callback = self . parse , dont_filter = True )


: param reason : : return : 所有url请求完毕之后关闭爬虫的回调函数

def close(self, reason):
        """
        :param reason:
        :return:所有url请求完毕之后关闭爬虫的回调函数
        """
        # 删除被删除的公众号 被删除的公众号content_url为空
        from db import delete
        delete(self.current_nickname, content_url="")
        print(self.name,"爬虫关闭")

def
close
(
self
,
reason
)
:
# 删除被删除的公众号 被删除的公众号content_url为空
from
db
import
delete
delete
(
self
.
current_nickname
,
content_url
=
""
)
print
(
self
.
name
,
"爬虫关闭")


def __init__(self, *args, **kwargs):
        """
        :param args:
        :param kwargs:
        实例化爬虫需要调用的函数
        """
        # 包含当前公众号所有不存在文本内容数据的生成器
        self.current_nickname = TidyReqData.get_nickname()
        self.articles_list = get_collection_article(self.current_nickname,article={"$exists": False},title={"$exists": True})
        self.crawler_begin_time = time()
        self.crawler_parse_counter = 0

def
__init__
(
self
,
*
args
,
*
*
kwargs
)
:
# 包含当前公众号所有不存在文本内容数据的生成器
self
.
current_nickname
=
TidyReqData
.
get_nickname
(
)
self
.
articles_list
=
get_collection_article
(
self
.
current_nickname
,
article
=
{
"$exists"
:
False
}
,
title
=
{
"$exists"
:
True
}
)
self
.
crawler_begin_time
=
time
(
)
self
.
crawler_parse_counter
=
0


def start_requests(self):
        """
        :return:重新爬虫的入口函数, 否者直接请求start_urls中的各个url
        重写之后手动调用Request并指定回调函数例如self.parse
        """
        for article in self.articles_list:
            if "weixin" in article['content_url']:
                yield Request(url=article['content_url'],callback=self.parse)

def start_requests ( self ) : for article in self . articles_list : if "weixin" in article [ 'content_url' ] : yield Request ( url = article [ 'content_url' ] , callback = self . parse )


def parse(self, response):
        """
        :param response:
        :return:请求完成之后的回调函数
        """
        item = CrawlArticleItem()
        item['article_data'] = response.get_ext_data['article_data']
        item['nickname'] = response.get_ext_data['nickname']
        item['raw_url'] = response.get_ext_data['raw_url']
        self.crawler_parse_counter += 1
        time_gap = time()-self.crawler_begin_time
        print(round(time_gap/self.crawler_parse_counter,3),item['article_data']['article'].replace('\n',''))
        # 发送状态给前端
        crawling_item = {}
        crawling_item['nickname'] = item['nickname']
        crawling_item['percent'] = self.crawler_parse_counter
        crawling_item['more'] = round(time_gap/self.crawler_parse_counter,3)
        crawling_item['title'] = find_one(item['nickname'],item['raw_url'])['title'][:10]
        gc.report_crawling(crawling_item)
        yield item

def
parse
(
self
,
response
)
:
item
=
CrawlArticleItem
(
)
item
[
'article_data'
]
=
response
.
get_ext_data
[
'article_data'
]
item
[
'nickname'
]
=
response
.
get_ext_data
[
'nickname'
]
item
[
'raw_url'
]
=
response
.
get_ext_data
[
'raw_url'
]
self
.
crawler_parse_counter
+=
1
time_gap
=
time
(
)
self
.
crawler_begin_time
print
(
round
(
time_gap
/
self
.
crawler_parse_counter
,
3
)
,
item
[
'article_data'
]
[
'article'
]
.
replace
(
'\n'
,
''
)
)
# 发送状态给前端
crawling_item
=
{
}
crawling_item
[
'nickname'
]
=
item
[
'nickname'
]
crawling_item
[
'percent'
]
=
self
.
crawler_parse_counter
crawling_item
[
'more'
]
=
round
(
time_gap
/
self
.
crawler_parse_counter
,
3
)
crawling_item
[
'title'
]
=
find_one
(
item
[
'nickname'
]
,
item
[
'raw_url'
]
)
[
'title'
]
[
:
10
]
gc
.
report_crawling
(
crawling_item
)
yield
item


def close(self, reason):
        """
        :param reason:
        :return:所有url请求完毕之后关闭爬虫的回调函数
        """
        time_gap = time()-self.crawler_begin_time
        if self.crawler_parse_counter != 0:
            print("%s爬虫关闭 用时%d 共计爬取%d 平均%f"%(self.name, time_gap, self.crawler_parse_counter, time_gap/self.crawler_parse_counter))
        from instance.global_instance import gs
        print("正在为 %s 创建索引..."%(self.current_nickname))
        index_result = gs.index_db_docs(self.current_nickname)
        print("索引完成",index_result)
        from db.meta_data import insert_article_metadata
        insert_article_metadata(self.current_nickname,{'date':datetime.datetime.now(),'articles_num':self.crawler_parse_counter})


def __init__(self, *args, **kwargs):
        """
        :param args:
        :param kwargs:
        实例化爬虫需要调用的函数
        """
        # 包含当前公众号所有不存在文本内容数据的生成器
        self.current_nickname = TidyReqData.get_nickname()
        print(self.current_nickname)
        articles_list = get_collection_article(self.current_nickname,read_num={"$exists": False},comment_id={"$exists": True})
        self.articles_list = []
        for article in articles_list:
            self.articles_list.append(article)
        self.task_num = len(self.articles_list)
        self.task_counter = 0
        self.begin_time = time()
        self.pre_time = time()

def
__init__
(
self
,
*
args
,
*
*
kwargs
)
:
# 包含当前公众号所有不存在文本内容数据的生成器
self
.
current_nickname
=
TidyReqData
.
get_nickname
(
)
print
(
self
.
current_nickname
)
articles_list
=
get_collection_article
(
self
.
current_nickname
,
read_num
=
{
"$exists"
:
False
}
,
comment_id
=
{
"$exists"
:
True
}
)
self
.
articles_list
=
[
]
for
article
in
articles_list
:
self
.
articles_list
.
append
(
article
)
self
.
task_num
=
len
(
self
.
articles_list
)
self
.
task_counter
=
0
self
.
begin_time
=
time
(
)
self
.
pre_time
=
time
(
)


def start_requests(self):
        """
        :return:重新爬虫的入口函数, 否者直接请求start_urls中的各个url
        重写之后手动调用Request并指定回调函数例如self.parse
        """
        for article in self.articles_list:
            if ':' in article['content_url']:
                request = Request(url=article['content_url'],callback=self.parse, dont_filter=False)
                request.set_ext_data({'content_url':article['content_url'],
                                      'comment_id':article['comment_id']})
                yield request

def start_requests ( self ) : for article in self . articles_list : if ':' in article [ 'content_url' ] : request = Request ( url = article [ 'content_url' ] , callback = self . parse , dont_filter = False ) request . set_ext_data ( { 'content_url' : article [ 'content_url' ] , 'comment_id' : article [ 'comment_id' ] } ) yield request


def parse(self, response):
        """
        :param response:
        :return:请求完成之后的回调函数
        """
        item = CrawlArticleReadDataItem()
        item['read_data'] = response.get_ext_data['read_data']
        item['nickname'] = response.get_ext_data['nickname']
        item['content_url'] = response.get_ext_data['content_url']
        # 打印状爬虫状态信息
        self.task_counter += 1
        pre_time_gap = time()-self.pre_time
        total_time_gap = time()-self.begin_time
        time_need = (self.task_num-self.task_counter)*(total_time_gap/self.task_counter)
        print(round(pre_time_gap,2),
              round(total_time_gap/self.task_counter,2),
              "%d/%d"%(self.task_counter,self.task_num),
              response.get_ext_data['read_data']['read_num'],
              response.get_ext_data['read_data']['like_num'],
              response.get_ext_data['read_data']['nick_name'],
              str(datetime.timedelta(seconds=time_need)).split('.')[0]
              )
        self.pre_time = time()
        crawling_item = {}
        crawling_item['nickname'] = item['nickname']
        crawling_item['percent'] = '%d/%d'%(self.task_counter,self.task_num)
        crawling_item['more'] = response.get_ext_data['read_data']['read_num']
        crawling_item['title'] = find_one(item['nickname'],item['content_url'])['title'][:10]
        gc.report_crawling(crawling_item)
        yield item

def
parse
(
self
,
response
)
:
item
=
CrawlArticleReadDataItem
(
)
item
[
'read_data'
]
=
response
.
get_ext_data
[
'read_data'
]
item
[
'nickname'
]
=
response
.
get_ext_data
[
'nickname'
]
item
[
'content_url'
]
=
response
.
get_ext_data
[
'content_url'
]
# 打印状爬虫状态信息
self
.
task_counter
+=
1
pre_time_gap
=
time
(
)
self
.
pre_time
total_time_gap
=
time
(
)
self
.
begin_time
time_need
=
(
self
.
task_num
self
.
task_counter
)
*
(
total_time_gap
/
self
.
task_counter
)
print
(
round
(
pre_time_gap
,
2
)
,
round
(
total_time_gap
/
self
.
task_counter
,
2
)
,
"%d/%d"
%
(
self
.
task_counter
,
self
.
task_num
)
,
response
.
get_ext_data
[
'read_data'
]
[
'read_num'
]
,
response
.
get_ext_data
[
'read_data'
]
[
'like_num'
]
,
response
.
get_ext_data
[
'read_data'
]
[
'nick_name'
]
,
str
(
datetime
.
timedelta
(
seconds
=
time_need
)
)
.
split
(
'.'
)
[
0
]
)
self
.
pre_time
=
time
(
)
crawling_item
=
{
}
crawling_item
[
'nickname'
]
=
item
[
'nickname'
]
crawling_item
[
'percent'
]
=
'%d/%d'
%
(
self
.
task_counter
,
self
.
task_num
)
crawling_item
[
'more'
]
=
response
.
get_ext_data
[
'read_data'
]
[
'read_num'
]
crawling_item
[
'title'
]
=
find_one
(
item
[
'nickname'
]
,
item
[
'content_url'
]
)
[
'title'
]
[
:
10
]
gc
.
report_crawling
(
crawling_item
)
yield
item


def close(self, reason):
        """
        :param reason:
        :return:所有url请求完毕之后关闭爬虫的回调函数
        """
        print(self.name,"爬虫关闭")

def close ( self , reason ) : print ( self . name , "爬虫关闭")


先转化为拼音形式再增加前缀

def _index_name(self, nickname):
        """
        :param nickname:
        :return: 根据公众号nickname的拼音产生专用的index名称（暂时没有添拼音相同的处理逻辑）, 先转化为拼音形式再增加前缀
        """
        return self.index_prefix+'_'+(to_pinyin_full(nickname)).lower()

def _index_name ( self , nickname ) : return self . index_prefix + '_' + ( to_pinyin_full ( nickname ) ) . lower ( )


def get_all_indices(self):
        """
        :return:获取es中公众号相关的所有index
        """
        pass


: param nickname : 公众号昵称的拼音 : return : 为一个公众号创建index 比如设置字段格式和指定搜索分词器 在index文档之前应该先mapping index 主要是进行文本分词字段设置

def create_index(self,nickname):
        """
        :param nickname:公众号昵称的拼音
        :return:为一个公众号创建index 比如设置字段格式和指定搜索分词器
        在index文档之前应该先mapping index 主要是进行文本分词字段设置
        """
        mapping_body = {}
        mapping_body['properties'] = doc_schema
        index_name = self._index_name(nickname)
        exists = es.indices.exists(index_name)
        if exists is False:
            es.indices.create(index_name)
            es.indices.put_mapping(index=self._index_name(nickname), doc_type=self.doc_type, body=mapping_body)
            logger.info('创建index %s 成功'%(nickname))
        else:
            logger.info('index %s 已经存在'%(nickname))
        return index_name

def
create_index
(
self
,
nickname
)
:
mapping_body
=
{
}
mapping_body
[
'properties'
]
=
doc_schema
index_name
=
self
.
_index_name
(
nickname
)
exists
=
es
.
indices
.
exists
(
index_name
)
if
exists
is
False
:
es
.
indices
.
create
(
index_name
)
es
.
indices
.
put_mapping
(
index
=
self
.
_index_name
(
nickname
)
,
doc_type
=
self
.
doc_type
,
body
=
mapping_body
)
logger
.
info
(
'创建index %s 成功'%(nickna
m
e
))
else
:
logger
.
info
(
'index %s 已经存在'%(nickna
m
e
))
return
index_name


: param nickname : 公众号昵称的拼音 * 删除所有index : return : 删除index和该index下的所有doc

def delete_index(self, nickname):
        """
        :param nickname: 公众号昵称的拼音 *删除所有index
        :return:删除index和该index下的所有doc
        """
        if nickname is not "*":
            index_name = self._index_name(nickname)
        es.indices.delete(index_name)

def delete_index ( self , nickname ) : if nickname is not "*" : index_name = self . _index_name ( nickname ) es . indices . delete ( index_name )


: param nickname : 公众号昵称 : return : 从mongodb中获取一个公众号的全部数据 使用bulk操作index进入es

def index_db_docs(self, nickname):
        """
        :param nickname:公众号昵称
        :return: 从mongodb中获取一个公众号的全部数据 使用bulk操作index进入es
        """
        # 先创建index
        index_name = self.create_index(nickname)
        # 从数据库中获取该公众号的全部文章
        articles = get_collection_article(nickname,article={"$exists": True},title={"$exists": True})
        articles_cache = []
        # mongodb的连接10分钟之后会过期 期间可能会出现完不成index的情况 故先将公众号的全部历史文章缓存
        for article in articles:
            doc = dict((key, article[key]) for key in doc_schema)
            articles_cache.append(doc)
        # 使用bulk操作index文档
        result = self.index_bulk(index_name,articles_cache)
        return result

# 从数据库中获取该公众号的全部文章


: param index_name : : param doc_dicts : : return :

def index_docs(self, index_name, doc_dicts):
        """
        :param index_name:
        :param doc_dicts:
        :return:
        """
        for doc_dict in doc_dicts:
            self.index_doc(index_name, doc_dict)


: param index_name : es中的index名称 : param doc_dict : 文档体 : return : 文档存在返回1 文档不存在返回0

def doc_exist(self, index_name, doc_dict):
        """
        :param index_name: es中的index名称
        :param doc_dict: 文档体
        :return:文档存在返回1 文档不存在返回0
        """
        # 使用文章的连接判断文档存在与否 存在数量为1 不存在数量为0
        try:
            body = {
                "query":{"match_phrase":{'content_url':doc_dict['content_url']}},
            }
            result = es.count(index=index_name, doc_type=self.doc_type, body=body)['count']
        # 如果公众号对应的index不存在则发生错误 也就说明该文章并未被创建索引
        except :
            result = 0
        return result

def
doc_exist
(
self
,
index_name
,
doc_dict
)
:
# 使用文章的连接判断文档存在与否 存在数量为1 不存在数量为0
try
:
body
=
{
"query"
:
{
"match_phrase"
:
{
'content_url'
:
doc_dict
[
'content_url'
]
}
}
,
}
result
=
es
.
count
(
index
=
index_name
,
doc_type
=
self
.
doc_type
,
body
=
body
)
[
'count'
]
# 如果公众号对应的index不存在则发生错误 也就说明该文章并未被创建索引
except
:
result
=
0
return
result


: param index_name : es中的index名称 : param doc_body : 文档体 : return : 新建或者更新doc

def index_doc(self, index_name, doc_dict):
        """
        :param index_name:es中的index名称
        :param doc_body:文档体
        :return:新建或者更新doc
        """
        doc = dict((key, doc_dict[key]) for key in doc_schema)
        # 当公众号文章更新之后还产生新的文章 直接跳过对旧有文章的index
        if self.doc_exist(index_name, doc_dict) == 1:
            return
        try:
            es.index(index=index_name, doc_type=self.doc_type, id=doc['content_url'], body=doc)
            logger.info('Index %d %s %s'%(doc['article_id'], index_name, doc['title']))
            print('Index %d %s %s'%(doc['article_id'], index_name, doc['title']))
        except:
            logger.warning('index 文档失败:%s %s'%(doc['nickname'], doc['title']))

def
index_doc
(
self
,
index_name
,
doc_dict
)
:
doc
=
dict
(
(
key
,
doc_dict
[
key
]
)
for
key
in
doc_schema
)
# 当公众号文章更新之后还产生新的文章 直接跳过对旧有文章的index
if
self
.
doc_exist
(
index_name
,
doc_dict
)
==
1
:
return
try
:
es
.
index
(
index
=
index_name
,
doc_type
=
self
.
doc_type
,
id
=
doc
[
'content_url'
]
,
body
=
doc
)
logger
.
info
(
'Index %d %s %s'
%
(
doc
[
'article_id'
]
,
index_name
,
doc
[
'title'
]
)
)
print
(
'Index %d %s %s'
%
(
doc
[
'article_id'
]
,
index_name
,
doc
[
'title'
]
)
)
except
:
logger
.
warning
(
'index 文档失败:%s %s'%(doc['n
i
c
kna
m
e'], doc['
t
i
le'
]
))


: param index_name : es中的index名称 : param doc_dict_list : 文档list doc包含了需要在es中建立索引的字段即可 也可包含更多字段但会过滤掉 : return : 使用bulk进行批量index API会根据指定的_id字段去重 而且支持更新

def index_bulk(self, index_name, doc_dict_list):
        """
        :param index_name: es中的index名称
        :param doc_dict_list: 文档list doc包含了需要在es中建立索引的字段即可 也可包含更多字段但会过滤掉
        :return:使用bulk进行批量index API会根据指定的_id字段去重 而且支持更新
        """
        if self.index_prefix not in index_name:
            index_name = self._index_name(index_name)
        actions = []
        for doc_dict in doc_dict_list:
            action = {
                "_index": index_name,
                "_type": self.doc_type,
                "_id": doc_dict['content_url'],
                "_source": doc_dict
            }
            actions.append(action)
        result = helpers.bulk(es, actions)
        return result

def index_bulk ( self , index_name , doc_dict_list ) : if self . index_prefix not in index_name : index_name = self . _index_name ( index_name ) actions = [ ] for doc_dict in doc_dict_list : action = { "_index" : index_name , "_type" : self . doc_type , "_id" : doc_dict [ 'content_url' ] , "_source" : doc_dict } actions . append ( action ) result = helpers . bulk ( es , actions ) return result


: param nickname : : param url : : return : 根据URL删除

def delete_doc(self, nickname, url):
        """
        :param nickname:
        :param url:
        :return: 根据URL删除
        """
        index_name = self._index_name(nickname)
        es.delete(index=index_name, doc_type=self.doc_type,id=url)

def delete_doc ( self , nickname , url ) : index_name = self . _index_name ( nickname ) es . delete ( index = index_name , doc_type = self . doc_type , id = url )


: param nicknames : 公众号昵称列表 : param search_data : 搜索字符串 : param source : 返回数据需要包含的字段 : return : 搜索

def search(self, nicknames, search_data, from_size={"from":0,"size":10}, source=None):
        """
        :param nicknames:公众号昵称列表
        :param search_data:搜索字符串
        :param source:返回数据需要包含的字段
        :return:搜索
        """
        # 根据公众号的昵称
        indices = []
        st = deepcopy(search_template)
        dls = self.search_data_preprocess(search_data)
        st.update(dls)
        if source != None:
            st["_source"] = source
        # 更新from 和 size 支持分页
        try:
            st["from"] = from_size["from"]
            st["size"] = from_size["size"]
        except:
            logger.warning("from_size字段错误 %s"%(str(from_size)))
        if nicknames == []:
            indices = 'gzh*'
        else:
            for nickname in nicknames:
                indices.append(self._index_name(nickname))

        try:
            result = es.search(index=indices, doc_type=self.doc_type, body=st)['hits']
            return result
        except Exception as e:
            logger.critical("搜索错误 可能是有部分公众号没有建立索引%s"%(str(indices)))
            return "ERROR"


: param nicknames : : param search_data : : param fields : : return : 返回搜索结果中的全部记录中的自定字段

def search_get_all(self,nicknames,search_data,source):
        """
        :param nicknames:
        :param search_data:
        :param fields:
        :return: 返回搜索结果中的全部记录中的自定字段
        """
        total = self.search(nicknames=nicknames, search_data=search_data,from_size={"from":0,"size":1},source=source)["total"]
        # 限制获取的数据量
        if total>=10000:
            total = 10000
        result_data = []
        hits = self.search(
            nicknames=nicknames,
            search_data=search_data,
            from_size={"from":0,"size":total},
            source=source)['hits']
        for hit in hits:
            result_data.append(hit['_source'])
        return result_data

def
search_get_all
(
self
,
nicknames
,
search_data
,
source
)
:
total
=
self
.
search
(
nicknames
=
nicknames
,
search_data
=
search_data
,
from_size
=
{
"from"
:
0
,
"size"
:
1
}
,
source
=
source
)
[
"total"
]
# 限制获取的数据量
if
total
>=
10000
:
total
=
10000
result_data
=
[
]
hits
=
self
.
search
(
nicknames
=
nicknames
,
search_data
=
search_data
,
from_size
=
{
"from"
:
0
,
"size"
:
total
}
,
source
=
source
)
[
'hits'
]
for
hit
in
hits
:
result_data
.
append
(
hit
[
'_source'
]
)
return
result_data


: param search_data : : return : 对即将搜索的数据进行预处理 解析搜索模式 数据中包含模式： 双引号包含的内容使用match_phrase全匹配 双引号之外的内容使用分词模式match 排序模式 指定排序字段以及升降方式 举例： 必须包含词 分词模式 - time - 1 根据搜索数据中指定的规则返回查询的query、sort等字段数据

def search_data_preprocess(self, search_data):
        """
        :param search_data:
        :return: 对即将搜索的数据进行预处理 解析搜索模式
        数据中包含模式：
        双引号包含的内容使用match_phrase全匹配,双引号之外的内容使用分词模式match
        排序模式 指定排序字段以及升降方式
        举例："必须包含词"分词模式-time-1
        根据搜索数据中指定的规则返回查询的query、sort等字段数据
        """
        sort_mapping = {
            "gzh":"nickname",       #公众号昵称
            "loc":"mov",            #文章发布位置
            "author":"author",      #作者
            "time":"p_date",        #发文时间
            "read":"read_num",      #阅读量
            "like":"like",          #点赞量
            "comm":"comments",      #评论量
            "reward":"reward_num",  #赞赏量
            "length":"article_token_len",#文章词数
            "unk":"_score",         #未知 就按照默认的分数排序
        }
        sort_dir_mapping = {
            '0':"asc",
            '1':"desc",
        }
        # 分离搜索数据 排序字段和排序顺序
        if len(re.findall('-',search_data))==2:
            part_data = search_data.split('-')
            try:
                sort_dir = sort_dir_mapping[part_data[-1]]
                sort_field = sort_mapping[part_data[-2]]
            except:
                sort_dir = sort_dir_mapping['1']
                sort_field = sort_mapping['unk']
        else:
            sort_dir = sort_dir_mapping['1']
            sort_field = sort_mapping['unk']
        search_data = search_data.split('-')[0]
        # 找出必须完整包含的字段
        data_match_phrase = [x.replace('"','') for x in re.findall(r'"\S*?"', search_data)]
        data_match = search_data.replace('"','')
        # 创建必须完整包含字段的Elsticsearch搜索描述数据
        for x in data_match_phrase:
            data_match = data_match.replace(x,'').replace(' ','')
        query_value = {
            "bool": {
                "must": []
            }
        }
        match_phrase_item = {
            "match_phrase": {"article": ""}
        }
        match_item = {
            "match": {"article": ""}
        }
        # 创建分词字段 英文不分词根据空格
        if data_match != '':
            match_item["match"]["article"] = data_match
            query_value["bool"]["must"].append(deepcopy(match_item))
        for item in data_match_phrase:
            match_phrase_item["match_phrase"]["article"]=item
            query_value["bool"]["must"].append(deepcopy(match_phrase_item))

        sort_value = [
            {
                sort_field: {
                    "order": sort_dir
                }
            }
        ]
        return {"query":query_value,"sort":sort_value}

def
search_data_preprocess
(
self
,
search_data
)
:
sort_mapping
=
{
"gzh"
:
"nickname"
,
#公众号昵称
"loc"
:
"mov"
,
#文章发布位置
"author"
:
"author"
,
#作者
"time"
:
"p_date"
,
#发文时间
"read"
:
"read_num"
,
#阅读量
"like"
:
"like"
,
#点赞量
"comm"
:
"comments"
,
#评论量
"reward"
:
"reward_num"
,
#赞赏量
"length"
:
"article_token_len"
,
#文章词数
"unk"
:
"_score"
,
#未知 就按照默认的分数排序
}
sort_dir_mapping
=
{
'0'
:
"asc"
,
'1'
:
"desc"
,
}
# 分离搜索数据 排序字段和排序顺序
if
len
(
re
.
findall
(
'-'
,
search_data
)
)
==
2
:
part_data
=
search_data
.
split
(
'-'
)
try
:
sort_dir
=
sort_dir_mapping
[
part_data
[
1
]
]
sort_field
=
sort_mapping
[
part_data
[
2
]
]
except
:
sort_dir
=
sort_dir_mapping
[
'1'
]
sort_field
=
sort_mapping
[
'unk'
]
else
:
sort_dir
=
sort_dir_mapping
[
'1'
]
sort_field
=
sort_mapping
[
'unk'
]
search_data
=
search_data
.
split
(
'-'
)
[
0
]
# 找出必须完整包含的字段
data_match_phrase
=
[
x
.
replace
(
'"'
,
''
)
for
x
in
re
.
findall
(
r'"\S*?"'
,
search_data
)
]
data_match
=
search_data
.
replace
(
'"'
,
''
)
# 创建必须完整包含字段的Elsticsearch搜索描述数据
for
x
in
data_match_phrase
:
data_match
=
data_match
.
replace
(
x
,
''
)
.
replace
(
' '
,
''
)
query_value
=
{
"bool"
:
{
"must"
:
[
]
}
}
match_phrase_item
=
{
"match_phrase"
:
{
"article"
:
""
}
}
match_item
=
{
"match"
:
{
"article"
:
""
}
}
# 创建分词字段 英文不分词根据空格
if
data_match
!=
''
:
match_item
[
"match"
]
[
"article"
]
=
data_match
query_value
[
"bool"
]
[
"must"
]
.
append
(
deepcopy
(
match_item
)
)
for
item
in
data_match_phrase
:
match_phrase_item
[
"match_phrase"
]
[
"article"
]
=
item
query_value
[
"bool"
]
[
"must"
]
.
append
(
deepcopy
(
match_phrase_item
)
)
sort_value
=
[
{
sort_field
:
{
"order"
:
sort_dir
}
}
]
return
{
"query"
:
query_value
,
"sort"
:
sort_value
}


: return : 返回所有的设置参数字典形式 以属性之一 : 搜索范围为例 当前选用的范围类型 每种范围类型下已经纳入的元素和尚可纳入的元素 setting_data = { search_range : { current_search_range : 公众号 or 类别 gzh_members : [ 爱迪斯 爱迪斯 爱迪斯 爱迪斯 ] gzh_available : [ 爱迪斯 爱迪斯 爱迪斯 爱迪斯 ] cat_members : [ 爱迪斯 爱迪斯 爱迪斯 爱迪斯 ] cat_available : [ 爱迪斯 爱迪斯 爱迪斯 爱迪斯 ] } }

def get_all_settings(self):
        """
        :return:返回所有的设置参数字典形式, 以属性之一:搜索范围为例
        当前选用的范围类型, 每种范围类型下已经纳入的元素和尚可纳入的元素
        setting_data = {
            'search_range':{
                'current_search_range':'公众号 or 类别',
                'gzh_members':['爱迪斯','爱迪斯','爱迪斯','爱迪斯'],
                'gzh_available':['爱迪斯','爱迪斯','爱迪斯','爱迪斯'],
                'cat_members':['爱迪斯','爱迪斯','爱迪斯','爱迪斯'],
                'cat_available':['爱迪斯','爱迪斯','爱迪斯','爱迪斯'],
            },
        }
        """
        setting_data = {}
        setting_data['search_range'] = {}
        # 获取全部公众号列表
        all_gzh_list = list(get_article_metadata().keys())
        all_gzh_list.append('全部')
        all_cat_list = []
        # 获得全部目录列表
        cats = DBQ.get_queue_by_kv(queue_type='公众号分类')
        for cat in cats:
            all_cat_list.append(cat['name'])
        # 获取当前正在使用的搜索范围方式
        try:
            if DBQ('公众号',self.queue_type).get_ext_data()['inuse'] is True:
                current_search_range = '公众号'
            else:
                current_search_range = '类别'
        except Exception as e:
            current_search_range = '公众号'
        # 获得现存搜索范围的公众号列表
        queue = DBQ.get_queue_by_kv(queue_type=self.queue_type,name='公众号')[0]['queue']
        gzh_members = []
        for element in queue:
            gzh_members.append(element['id'])
        gzh_available = sub_list(all_gzh_list, gzh_members)
        # 获得现存范围的类别列表
        queue = DBQ.get_queue_by_kv(queue_type=self.queue_type,name='类别')[0]['queue']
        cat_members = []
        for element in queue:
            cat_members.append(element['id'])
        cat_available = sub_list(all_cat_list, cat_members)

        setting_data['search_range']['current_search_range'] = current_search_range
        setting_data['search_range']['gzh_members'] = gzh_members
        setting_data['search_range']['gzh_available'] = gzh_available
        setting_data['search_range']['cat_members'] = cat_members
        setting_data['search_range']['cat_available'] = cat_available
        return setting_data

def
get_all_settings
(
self
)
:
setting_data
=
{
}
setting_data
[
'search_range'
]
=
{
}
# 获取全部公众号列表
all_gzh_list
=
list
(
get_article_metadata
(
)
.
keys
(
)
)
all_gzh_list
.
append
(
'全部')
all_cat_list
=
[
]
# 获得全部目录列表
cats
=
DBQ
.
get_queue_by_kv
(
queue_type
=
'公众号分类')
for
cat
in
cats
:
all_cat_list
.
append
(
cat
[
'name'
]
)
# 获取当前正在使用的搜索范围方式
try
:
if
DBQ
(
'公众号',self.
q
ueue
_
type).get_
e
x
t_data()['in
u
s
e
'] is T
r
e:
current_search_range
=
'公众号'
else
:
current_search_range
=
'类别'
except
Exception
as
e
:
current_search_range
=
'公众号'
# 获得现存搜索范围的公众号列表
queue
=
DBQ
.
get_queue_by_kv
(
queue_type
=
self
.
queue_type
,
name
=
'公众号')[0]['
q
u
e
u
e
']
gzh_members
=
[
]
for
element
in
queue
:
gzh_members
.
append
(
element
[
'id'
]
)
gzh_available
=
sub_list
(
all_gzh_list
,
gzh_members
)
# 获得现存范围的类别列表
queue
=
DBQ
.
get_queue_by_kv
(
queue_type
=
self
.
queue_type
,
name
=
'类别')[0]
[
'
q
u
e
ue']
cat_members
=
[
]
for
element
in
queue
:
cat_members
.
append
(
element
[
'id'
]
)
cat_available
=
sub_list
(
all_cat_list
,
cat_members
)
setting_data
[
'search_range'
]
[
'current_search_range'
]
=
current_search_range
setting_data
[
'search_range'
]
[
'gzh_members'
]
=
gzh_members
setting_data
[
'search_range'
]
[
'gzh_available'
]
=
gzh_available
setting_data
[
'search_range'
]
[
'cat_members'
]
=
cat_members
setting_data
[
'search_range'
]
[
'cat_available'
]
=
cat_available
return
setting_data


:param range_type:范围类型
        :return:更新搜索范围类型

:param range_type:范围类型
        :return:更新搜索范围类型

: param range_type : 范围类型 : return : 更新搜索范围类型

def change_search_range_type(self, range_type):
        """
        :param range_type:范围类型
        :return:更新搜索范围类型
        """
        DBQ('类别',self.queue_type).set_ext_data({'inuse':False})
        DBQ('公众号',self.queue_type).set_ext_data({'inuse':False})
        if range_type == '公众号':
            DBQ('公众号',self.queue_type).set_ext_data({'inuse':True})
        elif range_type == '类别':
            DBQ('类别',self.queue_type).set_ext_data({'inuse':True})
        return self.get_all_settings()

def change_search_range_type ( self , range_type ) : DBQ ( '类别',sel f .que u e_type).se t _ ext_data({'i n u se':Fal s e}) DBQ ( '公众号',self. q ueue _ type).set_ e x t_data({'inu s e ':False } ) if range_type == '公众号': DBQ ( '公众号',self. q ueue _ type).set_ e x t_data({'inu s e ':True} ) elif range_type == '类别': DBQ ( '类别',sel f .que u e_type).se t _ ext_data({'i n u se':Tru e }) return self . get_all_settings ( )


: param range_type : 范围类型 : param name : 公众号名称或者类别名称 : return : 从指定范围中删除元素 操作完成之后更新range_type

def delete_from_search_range(self, range_type, name):
        """
        :param range_type:范围类型
        :param name:公众号名称或者类别名称
        :return: 从指定范围中删除元素 操作完成之后更新range_type
        """
        if range_type == '公众号':
            DBQ('公众号',self.queue_type).delete_element({'id':name})
        elif range_type == '类别':
            DBQ('类别',self.queue_type).delete_element({'id':name})
        self.change_search_range_type(range_type)
        return self.get_all_settings()

def delete_from_search_range ( self , range_type , name ) : if range_type == '公众号': DBQ ( '公众号',self. q ueue _ type).dele t e _element({'id' : n ame} ) elif range_type == '类别': DBQ ( '类别',sel f .que u e_type).de l e te_element({'i d ' :nam e }) self . change_search_range_type ( range_type ) return self . get_all_settings ( )


: param range_type : 范围类型 : param name : 公众号名称或者类别名称 : return : 添加元素到指定范围 操作完成之后更新range_type

def add_to_search_range(self, range_type, name):
        """
        :param range_type:范围类型
        :param name:公众号名称或者类别名称
        :return: 添加元素到指定范围 操作完成之后更新range_type
        """
        if range_type == '公众号':
            gzh_queue = DBQ('公众号',self.queue_type)
            # 如果全部在搜索范围中直接返回
            # if '全部' in self.get_all_settings()['search_range']['gzh_members']:return
            # 如果在全部的公众号中搜索 则队列中只留下'全部'
            if name == '全部':
                gzh_queue.delete_all_element()
            gzh_queue.add_element({'id':name})
        elif range_type == '类别':
            DBQ('类别',self.queue_type).add_element({'id':name})
        self.change_search_range_type(range_type)
        return self.get_all_settings()

# 如果全部在搜索范围中直接返回

# 如果在全部的公众号中搜索 则队列中只留下'全部'


: parameter gc : GZHCategory实例用于获取存在目录的公众号成员列表 : return : 根据搜索范围设置参数 输出公众号集合列表

def search_range_data_preprocess(self,gc):
        """
        :parameter gc:GZHCategory实例用于获取存在目录的公众号成员列表
        :return:根据搜索范围设置参数 输出公众号集合列表
        """
        # 按照公众号
        search_range = self.get_all_settings()['search_range']
        if search_range['current_search_range'] == "公众号":
            if '全部' in search_range['gzh_members']:
                return []
            return search_range['gzh_members']
        # 按照分类
        elif search_range['current_search_range'] == "类别":
            gzhs = []
            search_cats = search_range['cat_members']
            all_cats = gc.get_all_cat_data()
            for member in search_cats:
                for cat in all_cats:
                    if member == cat['cat_name']:
                        gzhs = gzhs+cat['cat_members']
                        break
            return list(set(gzhs))
        # 全部
        else:
            return []

def
search_range_data_preprocess
(
self
,
gc
)
:
# 按照公众号
search_range
=
self
.
get_all_settings
(
)
[
'search_range'
]
if
search_range
[
'current_search_range'
]
==
"公众号":
if
'全部' in
ea
ch_range['gz
h
_members']:
return
[
]
return
search_range
[
'gzh_members'
]
# 按照分类
elif
search_range
[
'current_search_range'
]
==
"类别":
gzhs
=
[
]
search_cats
=
search_range
[
'cat_members'
]
all_cats
=
gc
.
get_all_cat_data
(
)
for
member
in
search_cats
:
for
cat
in
all_cats
:
if
member
==
cat
[
'cat_name'
]
:
gzhs
=
gzhs
+
cat
[
'cat_members'
]
break
return
list
(
set
(
gzhs
)
)
# 全部
else
:
return
[
]


: param result : : return : 格式化es搜索出来的结果，方便ui展示

def search_result_pretty(result):
    """
    :param result:
    :return:格式化es搜索出来的结果，方便ui展示
    """
    # from tools.utils import debug_p
    # debug_p(result)
    r = {}
    r['total'] = result['total']
    r['data'] = []
    for article in result['hits']:
        article['_source']['highlight'] = ''.join(article['highlight']['article']).replace('\n','')
        r['data'].append(article['_source'])
    return r

def
search_result_pretty
(
result
)
:
# from tools.utils import debug_p
# debug_p(result)
r
=
{
}
r
[
'total'
]
=
result
[
'total'
]
r
[
'data'
]
=
[
]
for
article
in
result
[
'hits'
]
:
article
[
'_source'
]
[
'highlight'
]
=
''
.
join
(
article
[
'highlight'
]
[
'article'
]
)
.
replace
(
'\n'
,
''
)
r
[
'data'
]
.
append
(
article
[
'_source'
]
)
return
r


: param raw_data : 来自search_get_all的返回数据 : param time_gap : 时间间隔 天 月 年三种单位 : return : 计算指定时间间隔内的文章数量 并生成二维数据用作图表可视化

def articles_and_time(raw_data, time_gap='month'):
    """
    :param raw_data:来自search_get_all的返回数据
    :param time_gap:时间间隔 天 月 年三种单位
    :return:计算指定时间间隔内的文章数量 并生成二维数据用作图表可视化
    """
    time_gap_mapping = {
        'year':['year'],
        'month':['year','month'],
        'day':['year','month','day'],
    }
    df = pd.DataFrame(columns=['year', 'month', 'day','week','tag'])
    for i in range(len(raw_data)):
        date_time = datetime.strptime(raw_data[i]['p_date'],'%Y-%m-%dT%H:%M:%S')
        row = p_date_extract(date_time)
        row.append(1)
        df.loc[i] = row
    df = df.groupby(time_gap_mapping[time_gap])[['tag']].describe(percentiles=[.5])
    # df分组之后抽取出纵坐标的参数
    y_data_list = df['tag']['count'].tolist()
    # df分组之后抽取出横坐标的参数
    x_data_list =['-'.join(x) for x in list(df.index.values)]
    return x_data_list,y_data_list

def
articles_and_time
(
raw_data
,
time_gap
=
'month'
)
:
time_gap_mapping
=
{
'year'
:
[
'year'
]
,
'month'
:
[
'year'
,
'month'
]
,
'day'
:
[
'year'
,
'month'
,
'day'
]
,
}
df
=
pd
.
DataFrame
(
columns
=
[
'year'
,
'month'
,
'day'
,
'week'
,
'tag'
]
)
for
i
in
range
(
len
(
raw_data
)
)
:
date_time
=
datetime
.
strptime
(
raw_data
[
i
]
[
'p_date'
]
,
'%Y-%m-%dT%H:%M:%S'
)
row
=
p_date_extract
(
date_time
)
row
.
append
(
1
)
df
.
loc
[
i
]
=
row
df
=
df
.
groupby
(
time_gap_mapping
[
time_gap
]
)
[
[
'tag'
]
]
.
describe
(
percentiles
=
[
.5
]
)
# df分组之后抽取出纵坐标的参数
y_data_list
=
df
[
'tag'
]
[
'count'
]
.
tolist
(
)
# df分组之后抽取出横坐标的参数
x_data_list
=
[
'-'
.
join
(
x
)
for
x
in
list
(
df
.
index
.
values
)
]
return
x_data_list
,
y_data_list


: param p_date : datetime实例 : return : [ 年 月 日 星期 ]

def p_date_extract(p_date):
    """
    :param p_date: datetime实例
    :return:['年','月','日','星期']
    """
    date = p_date.strftime("%Y-%m-%d").split('-')
    date.append(str(p_date.weekday()+1))
    return date

def p_date_extract ( p_date ) : date = p_date . strftime ( "%Y-%m-%d" ) . split ( '-' ) date . append ( str ( p_date . weekday ( ) + 1 ) ) return date


将混合转化为搜索拼音字符

def to_pinyin(ch_name):
    """
    将混合转化为搜索拼音字符
    INPUT '杭州frank1湖滨银泰in77'
    OUTPUT 'hzfrank1hbytin77'
    """
    from pypinyin import lazy_pinyin
    import re
    d1 = re.sub(r'[a-zA-Z0-9]+',' ',ch_name)
    d2 = re.findall(r'[a-zA-Z0-9]+',ch_name)
    d3 = lazy_pinyin(d1)
    d4 = []
    i=0
    for d in d3:
        if d == ' ':
            d4.append(d2[i])
            i+=1
        else:
            d4.append(d[0])
    d4 = ''.join(d4)
    return d4

def
to_pinyin
(
ch_name
)
:
from
pypinyin
import
lazy_pinyin
import
re
d1
=
re
.
sub
(
r'[a-zA-Z0-9]+'
,
' '
,
ch_name
)
d2
=
re
.
findall
(
r'[a-zA-Z0-9]+'
,
ch_name
)
d3
=
lazy_pinyin
(
d1
)
d4
=
[
]
i
=
0
for
d
in
d3
:
if
d
==
' '
:
d4
.
append
(
d2
[
i
]
)
i
+=
1
else
:
d4
.
append
(
d
[
0
]
)
d4
=
''
.
join
(
d4
)
return
d4


key与value通过split_symbol连接， key value 对之间使用join_symbol连接 例如： a = b&c = d join_symbol是& split_symbol是 = : param s : 原字符串 : param join_symbol : 连接符 : param split_symbol : 分隔符 : return : 字典

def str_to_dict(s, join_symbol="\n", split_symbol=":"):
    """
    key与value通过split_symbol连接， key,value 对之间使用join_symbol连接
    例如： a=b&c=d   join_symbol是&, split_symbol是=
    :param s: 原字符串
    :param join_symbol: 连接符
    :param split_symbol: 分隔符
    :return: 字典
    """
    s_list = s.split(join_symbol)
    data = dict()
    for item in s_list:
        item = item.strip()
        if item:
            k, v = item.split(split_symbol, 1)
            data[k] = v.strip()
    return data

def str_to_dict ( s , join_symbol = "\n" , split_symbol = ":" ) : s_list = s . split ( join_symbol ) data = dict ( ) for item in s_list : item = item . strip ( ) if item : k , v = item . split ( split_symbol , 1 ) data [ k ] = v . strip ( ) return data


: param whole_list : : param part_list : : return : 从whole list中找出除了part_list之外的元素 组成一个子list返回

def sub_list(whole_list, part_list):
    """
    :param whole_list:
    :param part_list:
    :return:从whole list中找出除了part_list之外的元素 组成一个子list返回
    """
    return [x for x in whole_list if x not in part_list]

def sub_list ( whole_list , part_list ) : return [ x for x in whole_list if x not in part_list ]


: param q_name : 创建一个队列 开头处插入一个__BEGIN 所有队列名称均以re__开头

def __init__(self, q_name):
        """
        :param q_name:创建一个队列 开头处插入一个__BEGIN
        所有队列名称均以re__开头
        """
        self.q_name = 'rq__'+q_name
        self.redis = redis_instance

def __init__ ( self , q_name ) : self . q_name = 'rq__' + q_name self . redis = redis_instance


:param data:
        :return:1表示插入成功 1表示对象已经存在

:param data:
        :return:1表示插入成功 1表示对象已经存在

: param data : : return : 1表示插入成功 1表示对象已经存在

def push(self, data):
        """
        :param data:
        :return:1表示插入成功 1表示对象已经存在
        """
        rq = self.get_rq_data()
        if data not in rq:
            self.redis.lpush(self.q_name,data)
            return 1
        return 0

def push ( self , data ) : rq = self . get_rq_data ( ) if data not in rq : self . redis . lpush ( self . q_name , data ) return 1 return 0


def pop(self):
        """
        :return:[]表示队列已经空了
        """
        data = self.redis.rpop(self.q_name)
        try:
            rq_j_data = json.loads(data)
        except:
            if data:
                rq_j_data = data.decode('utf8')
            else:
                rq_j_data = []
        return rq_j_data

def pop ( self ) : data = self . redis . rpop ( self . q_name ) try : rq_j_data = json . loads ( data ) except : if data : rq_j_data = data . decode ( 'utf8' ) else : rq_j_data = [ ] return rq_j_data


: param data : 根据data删除指定的元素 : return : 删除后的队列

def remove(self,data):
        """
        :param data:根据data删除指定的元素
        :return:删除后的队列
        """
        rq_list = self.get_rq_data()
        self.delete_rq()
        for item in reversed(rq_list):
            if item is not data:
                self.push(item)
        rq_list = self.get_rq_data()
        return rq_list

def remove ( self , data ) : rq_list = self . get_rq_data ( ) self . delete_rq ( ) for item in reversed ( rq_list ) : if item is not data : self . push ( item ) rq_list = self . get_rq_data ( ) return rq_list


def get_rq_data(self):
        """
        :return:返回插入的数据
        """
        rq_b_data_list = self.redis.lrange(self.q_name,0,-1)
        rq_j_data_list = []
        for rq_b_data in rq_b_data_list:
            try:
                rq_j_data = json.loads(rq_b_data)
            except:
                rq_j_data = rq_b_data.decode('utf8')
            rq_j_data_list.append(rq_j_data)
        return rq_j_data_list

def get_rq_data ( self ) : rq_b_data_list = self . redis . lrange ( self . q_name , 0 , - 1 ) rq_j_data_list = [ ] for rq_b_data in rq_b_data_list : try : rq_j_data = json . loads ( rq_b_data ) except : rq_j_data = rq_b_data . decode ( 'utf8' ) rq_j_data_list . append ( rq_j_data ) return rq_j_data_list


: param name : 队列名称 : param queue_type : 队列类型 通常用于标识用途

def __init__(self, name, queue_type):
        """
        :param name:队列名称
        :param queue_type:队列类型 通常用于标识用途
        """
        self.name = name
        self.queue_type = queue_type
        self.queue_structure = copy(data_queue_scheme)
        # name, queue_type唯一标识一个queue
        self.queue_structure['name'] = name
        self.queue_structure['queue_type'] = queue_type
        # 初始化之后创建一个空的队列
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        if type(queue_data) is not dict:
            queue_data = copy(self.queue_structure)
            DBQ.col.insert_one(queue_data)

def
__init__
(
self
,
name
,
queue_type
)
:
self
.
name
=
name
self
.
queue_type
=
queue_type
self
.
queue_structure
=
copy
(
data_queue_scheme
)
# name, queue_type唯一标识一个queue
self
.
queue_structure
[
'name'
]
=
name
self
.
queue_structure
[
'queue_type'
]
=
queue_type
# 初始化之后创建一个空的队列
queue_data
=
DBQ
.
col
.
find_one
(
{
'name'
:
self
.
name
,
'queue_type'
:
self
.
queue_type
}
)
if
type
(
queue_data
)
is
not
dict
:
queue_data
=
copy
(
self
.
queue_structure
)
DBQ
.
col
.
insert_one
(
queue_data
)


: param element : 元素数据 elemet是一个dict 必须指定id属性 : return : 增加一个元素

def add_element(self, element):
        """
        :param element:元素数据 elemet是一个dict 必须指定id属性
        :return:增加一个元素
        """
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        # 更新
        if type(queue_data) is dict:
            self.update_element(element)
        # 插入
        else:
            queue_data = copy(self.queue_structure)
            queue_data['update_time'] = datetime.now()
            queue_data['length'] = 1
            queue_data['queue'] = [element]
            DBQ.col.insert_one(queue_data)
        return queue_data

def
add_element
(
self
,
element
)
:
queue_data
=
DBQ
.
col
.
find_one
(
{
'name'
:
self
.
name
,
'queue_type'
:
self
.
queue_type
}
)
# 更新
if
type
(
queue_data
)
is
dict
:
self
.
update_element
(
element
)
# 插入
else
:
queue_data
=
copy
(
self
.
queue_structure
)
queue_data
[
'update_time'
]
=
datetime
.
now
(
)
queue_data
[
'length'
]
=
1
queue_data
[
'queue'
]
=
[
element
]
DBQ
.
col
.
insert_one
(
queue_data
)
return
queue_data


: param element : 元素数据 : return : 根据data中的id删除该元素 1表示产出成功 0表示未找到匹配id的元素

def delete_element(self, element):
        """
        :param element:元素数据
        :return: 根据data中的id删除该元素
        1表示产出成功
        0表示未找到匹配id的元素
        """
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})

        elements = queue_data['queue']
        for ele in elements:
            if ele['id'] == element['id']:
                elements.remove(ele)
                queue_data['update_time'] = datetime.now()
                queue_data['length'] -= 1
                DBQ.col.update_one({'name':self.name, 'queue_type':self.queue_type},{"$set":queue_data})
                return 1
        return 0

def delete_element ( self , element ) : queue_data = DBQ . col . find_one ( { 'name' : self . name , 'queue_type' : self . queue_type } ) elements = queue_data [ 'queue' ] for ele in elements : if ele [ 'id' ] == element [ 'id' ] : elements . remove ( ele ) queue_data [ 'update_time' ] = datetime . now ( ) queue_data [ 'length' ] -= 1 DBQ . col . update_one ( { 'name' : self . name , 'queue_type' : self . queue_type } , { "$set" : queue_data } ) return 1 return 0


: param element : 元素数据 : return : 根据data中的id更新该元素

def update_element(self, element):
        """
        :param element: 元素数据
        :return:根据data中的id更新该元素
        """
        # 得到队列 如果存在返回dict否则返回None
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        old_list = queue_data['queue']
        # 根据elemet更新队列
        result = DBQ.update_dict_list_by_kv(old_list=old_list,element=element)
        queue_data['queue'] = old_list
        queue_data['update_time'] = datetime.now()
        # 如果是新增元素长度自增1
        if result == 2:
            queue_data['length'] += 1
        DBQ.col.update_one({'name':self.name, 'queue_type':self.queue_type},{"$set":queue_data})

def
update_element
(
self
,
element
)
:
# 得到队列 如果存在返回dict否则返回None
queue_data
=
DBQ
.
col
.
find_one
(
{
'name'
:
self
.
name
,
'queue_type'
:
self
.
queue_type
}
)
old_list
=
queue_data
[
'queue'
]
# 根据elemet更新队列
result
=
DBQ
.
update_dict_list_by_kv
(
old_list
=
old_list
,
element
=
element
)
queue_data
[
'queue'
]
=
old_list
queue_data
[
'update_time'
]
=
datetime
.
now
(
)
# 如果是新增元素长度自增1
if
result
==
2
:
queue_data
[
'length'
]
+=
1
DBQ
.
col
.
update_one
(
{
'name'
:
self
.
name
,
'queue_type'
:
self
.
queue_type
}
,
{
"$set"
:
queue_data
}
)


:return:清空队列中的元素

:return:清空队列中的元素

: return : 清空队列中的元素

def delete_all_element(self):
        """
        :return:清空队列中的元素
        """
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        queue_data['queue'] = []
        DBQ.col.update_one({'name':self.name, 'queue_type':self.queue_type},{"$set":queue_data})

def delete_all_element ( self ) : queue_data = DBQ . col . find_one ( { 'name' : self . name , 'queue_type' : self . queue_type } ) queue_data [ 'queue' ] = [ ] DBQ . col . update_one ( { 'name' : self . name , 'queue_type' : self . queue_type } , { "$set" : queue_data } )


def delete_self(self):
        """
        :return:删除队列自己
        """
        DBQ.col.delete_one({'name':self.name, 'queue_type':self.queue_type})

def delete_self ( self ) : DBQ . col . delete_one ( { 'name' : self . name , 'queue_type' : self . queue_type } )


def get_queue(self):
        """
        :return:获得queue list数据
        """
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        return queue_data['queue']

def get_queue ( self ) : queue_data = DBQ . col . find_one ( { 'name' : self . name , 'queue_type' : self . queue_type } ) return queue_data [ 'queue' ]


:param ext_data:字典
        :return: 设置ext_data

:param ext_data:字典
        :return: 设置ext_data

: param ext_data : 字典 : return : 设置ext_data

def set_ext_data(self, ext_data):
        """
        :param ext_data:字典
        :return: 设置ext_data
        """
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        queue_data['ext_data'] = ext_data
        DBQ.col.update_one({'name':self.name, 'queue_type':self.queue_type},
                           {"$set":queue_data})

def set_ext_data ( self , ext_data ) : queue_data = DBQ . col . find_one ( { 'name' : self . name , 'queue_type' : self . queue_type } ) queue_data [ 'ext_data' ] = ext_data DBQ . col . update_one ( { 'name' : self . name , 'queue_type' : self . queue_type } , { "$set" : queue_data } )


def get_ext_data(self):
        """
        :return: 获取ext_data字典
        """
        queue_data = DBQ.col.find_one({'name':self.name, 'queue_type':self.queue_type})
        if type(queue_data) is dict:
            return queue_data['ext_data']
        return None

def get_ext_data ( self ) : queue_data = DBQ . col . find_one ( { 'name' : self . name , 'queue_type' : self . queue_type } ) if type ( queue_data ) is dict : return queue_data [ 'ext_data' ] return None


: param key : : param value : : return : 根据给定字典中的某个键值对跟新所在list 以保证list中元素的唯一性 返回0表示失败 返回1表示更新 返回2表示增加

def update_dict_list_by_kv(old_list, element, key='id'):
        """
        :param key:
        :param value:
        :return:根据给定字典中的某个键值对跟新所在list, 以保证list中元素的唯一性
        返回0表示失败
        返回1表示更新
        返回2表示增加
        """
        if key not in element:
            logger.warning("key不存在更新list失败%s"%(str(key)))
            return 0
        for ele in old_list:
            if ele[key] == element[key]:
                old_list[old_list.index(ele)] = element
                return 1
        old_list.append(element)
        return 2

def update_dict_list_by_kv ( old_list , element , key = 'id' ) : if key not in element : logger . warning ( "key不存在更新list失败%s"%(str(key))) return 0 for ele in old_list : if ele [ key ] == element [ key ] : old_list [ old_list . index ( ele ) ] = element return 1 old_list . append ( element ) return 2


:param valye:
        :return:通过特定的键值对组合获取合适的队列

:param valye:
        :return:通过特定的键值对组合获取合适的队列

: param valye : : return : 通过特定的键值对组合获取合适的队列

def get_queue_by_kv(cls, **kwargs):
        """
        :param valye:
        :return:通过特定的键值对组合获取合适的队列
        """
        queue_gen = DBQ.col.find(kwargs)
        queue_list = []
        for queue in queue_gen:
            queue_list.append(queue)
        return queue_list

def get_queue_by_kv ( cls , * * kwargs ) : queue_gen = DBQ . col . find ( kwargs ) queue_list = [ ] for queue in queue_gen : queue_list . append ( queue ) return queue_list


def delete_all_queue(cls):
        """
        :return:删除所有队列
        """
        DBQ.col.delete_many()

def delete_all_queue ( cls ) : DBQ . col . delete_many ( )


: param name : 队列名称 : param queue_type : 队列类型 : return : 获取指定名称和类型的队列

def delete_queue(cls,name,queue_type):
        """
        :param name: 队列名称
        :param queue_type: 队列类型
        :return: 获取指定名称和类型的队列
        """
        DBQ.col.delete_one({'name':name,'queue_type':queue_type})

def delete_queue ( cls , name , queue_type ) : DBQ . col . delete_one ( { 'name' : name , 'queue_type' : queue_type } )


: param nickname : 更新的公众号昵称 : param update_log : 更新详细日志 : return : 一个公众号占用一行记录

def insert_article_metadata(nickname, update_log):
    """
    :param nickname: 更新的公众号昵称
    :param update_log: 更新详细日志
    :return: 一个公众号占用一行记录
    """
    metadata = col.find_one({'nickname':nickname})
    if type(metadata) is dict:
        metadata['update_log_list'].append(update_log)
        col.update_one({'nickname':nickname},{"$set":metadata})
    # metadata不存在
    else:
        metadata = copy(meta_data_scheme)
        metadata['nickname'] = nickname
        metadata['update_log_list'] = [update_log]
        col.insert_one(metadata)

def
insert_article_metadata
(
nickname
,
update_log
)
:
metadata
=
col
.
find_one
(
{
'nickname'
:
nickname
}
)
if
type
(
metadata
)
is
dict
:
metadata
[
'update_log_list'
]
.
append
(
update_log
)
col
.
update_one
(
{
'nickname'
:
nickname
}
,
{
"$set"
:
metadata
}
)
# metadata不存在
else
:
metadata
=
copy
(
meta_data_scheme
)
metadata
[
'nickname'
]
=
nickname
metadata
[
'update_log_list'
]
=
[
update_log
]
col
.
insert_one
(
metadata
)


: param nickname : 指定账号的日志 None为全部账号 : param all : True全部日志 False : 最后一次更新日志 : return : 获取公众号文章平爬取的日志信息

def get_article_metadata(nickname=None, all=True):
    """
    :param nickname:指定账号的日志 None为全部账号
    :param all:True全部日志 False:最后一次更新日志
    :return: 获取公众号文章平爬取的日志信息
    """
    data = {}
    metadata = []
    if nickname is not None:
        metadata = [col.find_one({'nickname':nickname})]
    else:
        for md in col.find():
            metadata.append(md)
    for item in metadata:
        if all:data[item['nickname']] = item['update_log_list']
        else :data[item['nickname']] = item['update_log_list'][-1]
    return data

def get_article_metadata ( nickname = None , all = True ) : data = { } metadata = [ ] if nickname is not None : metadata = [ col . find_one ( { 'nickname' : nickname } ) ] else : for md in col . find ( ) : metadata . append ( md ) for item in metadata : if all : data [ item [ 'nickname' ] ] = item [ 'update_log_list' ] else : data [ item [ 'nickname' ] ] = item [ 'update_log_list' ] [ - 1 ] return data


: param nickname : 公众号昵称 : return : 删除指定公众号的微信爬取日志信息

def delete_article_metadata(nickname):
    """
    :param nickname: 公众号昵称
    :return: 删除指定公众号的微信爬取日志信息
    """
    col.delete_one({'nickname':nickname})

def delete_article_metadata ( nickname ) : col . delete_one ( { 'nickname' : nickname } )


def update_history():
    """
    :return:为此前通过手动爬取的文章创建metadata
    """
    from db import WeixinDB
    from datetime import datetime
    wxdb = WeixinDB()
    nicknames = wxdb.get_all_nickname(is_count=True)
    for item in nicknames:
        if item[0] not in ['queue','crawler_metadata']:
            insert_article_metadata(item[0],{'date':datetime.now(),'articles_num':item[1]})

def update_history ( ) : from db import WeixinDB from datetime import datetime wxdb = WeixinDB ( ) nicknames = wxdb . get_all_nickname ( is_count = True ) for item in nicknames : if item [ 0 ] not in [ 'queue' , 'crawler_metadata' ] : insert_article_metadata ( item [ 0 ] , { 'date' : datetime . now ( ) , 'articles_num' : item [ 1 ] } )


: param article_seg : 含有一条article部分或者全部的数据 : return : 根据article_scheme中指定的字段提取article_seg中的数据

def update_article_from_template(article_seg):
    """
    :param article_seg: 含有一条article部分或者全部的数据
    :return:根据article_scheme中指定的字段提取article_seg中的数据
    """
    article = {}
    for key in article_scheme:
        if key in article_seg:
            article[key] = article_seg[key]
    return article

def update_article_from_template ( article_seg ) : article = { } for key in article_scheme : if key in article_seg : article [ key ] = article_seg [ key ] return article


: param nickname : : param article : dict data : return : 在nickname collection中插入一公众号文章数据

def insert_one(nickname, article):
    """
    :param nickname:
    :param article: dict data
    :return: 在nickname collection中插入一公众号文章数据
    """
    col = db_instance[nickname]
    return col.insert_one(article).inserted_id

def insert_one ( nickname , article ) : col = db_instance [ nickname ] return col . insert_one ( article ) . inserted_id


: param nickname : : param articel : : return : 根据article中的content_url更新文章数据

def update_one(nickname, articel):
    """
    :param nickname:
    :param articel:
    :return: 根据article中的content_url更新文章数据
    """
    op_result = ''
    # 已经删除的文章不存在content_url属性 直接返回局
    if articel["content_url"] is "":
        op_result = 'ERROR'
        return op_result
    col = db_instance[nickname]
    result = col.find_one({"content_url":articel['content_url']})
    if type(result) is dict:
        # 数据存在可以更新
        col.update_one({"content_url":articel['content_url']},
                       {"$set":articel})
        op_result = 'UPDATE'
    else:
        # 数据不存在调用插入
        insert_one(nickname, articel)
        op_result = 'INSERT'
    return op_result

def
update_one
(
nickname
,
articel
)
:
op_result
=
''
# 已经删除的文章不存在content_url属性 直接返回局
if
articel
[
"content_url"
]
is
""
:
op_result
=
'ERROR'
return
op_result
col
=
db_instance
[
nickname
]
result
=
col
.
find_one
(
{
"content_url"
:
articel
[
'content_url'
]
}
)
if
type
(
result
)
is
dict
:
# 数据存在可以更新
col
.
update_one
(
{
"content_url"
:
articel
[
'content_url'
]
}
,
{
"$set"
:
articel
}
)
op_result
=
'UPDATE'
else
:
# 数据不存在调用插入
insert_one
(
nickname
,
articel
)
op_result
=
'INSERT'
return
op_result


: param nickname : : param articles : article lsit : param check_exist : 如否需要根据content_url判重 : return : 在nickname collection中插入一个list的文章

def insert_many(nickname, articles, check_exist=True):
    """
    :param nickname:
    :param articles:article lsit
    :param check_exist:如否需要根据content_url判重
    :return: 在nickname collection中插入一个list的文章
    """
    # 是否存在被更新的记录
    has_update = False
    if check_exist==False:
        col = db_instance[nickname]
        col.insert_many(articles)
    else:
        for article in articles:
            result = update_one(nickname, article)
            if result == 'UPDATE':
                has_update = True
    return has_update

def
insert_many
(
nickname
,
articles
,
check_exist
=
True
)
:
# 是否存在被更新的记录
has_update
=
False
if
check_exist
==
False
:
col
=
db_instance
[
nickname
]
col
.
insert_many
(
articles
)
else
:
for
article
in
articles
:
result
=
update_one
(
nickname
,
article
)
if
result
==
'UPDATE'
:
has_update
=
True
return
has_update


:param nickname:
    :return: 计算当前公众号的文章数量

:param nickname:
    :return: 计算当前公众号的文章数量

: param nickname : : return : 计算当前公众号的文章数量

def count(nickname):
    """
    :param nickname:
    :return: 计算当前公众号的文章数量
    """
    return db_instance[nickname].count()

def count ( nickname ) : return db_instance [ nickname ] . count ( )


: param nickname : : param kwargs : 用字典表示的过滤器 : return : 根据match中提供的符合信息删除文章 支持全部删除

def delete(nickname, **kwargs):
    """
    :param nickname:
    :param kwargs: 用字典表示的过滤器
    :return: 根据match中提供的符合信息删除文章 支持全部删除
    """
    col = db_instance[nickname]
    col.delete_many(kwargs)

def delete ( nickname , * * kwargs ) : col = db_instance [ nickname ] col . delete_many ( kwargs )


: param nickname : : param content_url : : return :

def find_one(nickname,content_url):
    """
    :param nickname:
    :param content_url:
    :return:
    """
    col = db_instance[nickname]
    result = col.find_one({'content_url':content_url})
    return result

def find_one ( nickname , content_url ) : col = db_instance [ nickname ] result = col . find_one ( { 'content_url' : content_url } ) return result


:param nickname:
    :return: 删除collection

:param nickname:
    :return: 删除collection

: param nickname : : return : 删除collection

def drop_collection(nickname):
    """
    :param nickname:
    :return: 删除collection
    """
    db_instance.drop_collection(nickname)
    return nickname

def drop_collection ( nickname ) : db_instance . drop_collection ( nickname ) return nickname


: param nickname : : param kwargs : 例如找出不存在article字段的所有数据 { article : { $exists : False }} : return : 以生成器的形式返回一个公众号的全部或者部分数据

def get_collection_article(nickname,**kwargs):
    """
    :param nickname:
    :param kwargs: 例如找出不存在article字段的所有数据 {"article":{"$exists": False}}
    :return: 以生成器的形式返回一个公众号的全部或者部分数据
    """
    col = db_instance[nickname]
    articles = col.find(kwargs)
    for article in articles:
        yield article

def get_collection_article ( nickname , * * kwargs ) : col = db_instance [ nickname ] articles = col . find ( kwargs ) for article in articles : yield article