#NEWLINE# def crawl( self , crawl_candidate ) : #NEWLINE# #INDENT# article = Article ( ) #NEWLINE# parse_candidate = self . get_parse_candidate ( crawl_candidate ) #NEWLINE# raw_html = self . get_html ( crawl_candidate , parse_candidate ) #NEWLINE# if ( raw_html is None ) : #NEWLINE# #INDENT# return article #NEWLINE# #UNINDENT# doc = self . get_document ( raw_html ) #NEWLINE# extractor = self . get_extractor ( ) #NEWLINE# document_cleaner = self . get_document_cleaner ( ) #NEWLINE# output_formatter = self . get_output_formatter ( ) #NEWLINE# article . final_url = parse_candidate . url #NEWLINE# article . link_hash = parse_candidate . link_hash #NEWLINE# article . raw_html = raw_html #NEWLINE# article . doc = doc #NEWLINE# article . raw_doc = deepcopy ( doc ) #NEWLINE# article . title = extractor . get_title ( article ) #NEWLINE# article . meta_lang = extractor . get_meta_lang ( article ) #NEWLINE# article . meta_favicon = extractor . get_favicon ( article ) #NEWLINE# article . meta_description = extractor . get_meta_description ( article ) #NEWLINE# article . meta_keywords = extractor . get_meta_keywords ( article ) #NEWLINE# article . canonical_link = extractor . get_canonical_link ( article ) #NEWLINE# article . domain = extractor . get_domain ( article . final_url ) #NEWLINE# article . tags = extractor . extract_tags ( article ) #NEWLINE# article . doc = document_cleaner . clean ( article ) #NEWLINE# article . top_node = extractor . calculate_best_node ( article ) #NEWLINE# if ( article . top_node is not None ) : #NEWLINE# #INDENT# video_extractor = self . get_video_extractor ( article ) #NEWLINE# video_extractor . get_videos ( ) #NEWLINE# if self . config . enable_image_fetching : #NEWLINE# #INDENT# image_extractor = self . get_image_extractor ( article ) #NEWLINE# article . top_image = image_extractor . get_best_image ( article . raw_doc , article . top_node ) #NEWLINE# #UNINDENT# article . top_node = extractor . post_cleanup ( article . top_node ) #NEWLINE# article . cleaned_text = output_formatter . get_formatted_text ( article ) #NEWLINE# #UNINDENT# self . release_resources ( article ) #NEWLINE# return article