Package: crawler-rocket
Crawler Rocket :rocket:Why build this ？:grey_question:As an information analyst in the library of my university, I usually need to write a lot of crawler scripts with all kinds of functions, but I found that I spent a lot of time writing non-business logic code such as environment configuration or document processing(excel, csv...), so I developed this tool to help myself quickly build the basic framework of crawlers and handle some redundant work, so that I can focus on writing real business logic.How to Use this? :muscle:Here's a code example from my previous jobimporttracebackfromseleniumimportwebdriverfromselenium.webdriver.common.byimportByfromselenium.commonimportexceptionsimporttimeimportxlsxwriterimportosclassShangHaiRanking:__SHANGHAIRANKING_ROOT='./科软排名'indicator_list=[]more_indicator_cnt=0def__init__(self):self._driver=webdriver.Chrome()def__initIndicatorList(self):self.indicator_list.clear()butt=self._driver.find_element_by_xpath('//*[@id="content-box"]/div[2]/table/thead/tr/th[6]/div/div[1]/div[1]/input')self._driver.execute_script('arguments[0].click();',butt)menu=self._driver.find_element_by_xpath('//*[@id="content-box"]/div[2]/table/thead/tr/th[6]/div/div[1]/div[2]/ul')\.find_elements_by_css_selector('li')forminmenu:self.indicator_list.append(m.text)self.more_indicator_cnt=len(menu)#init workbookdef__initWorkBook(self,year):ifnotos.path.exists(self.__SHANGHAIRANKING_ROOT):os.mkdir(self.__SHANGHAIRANKING_ROOT)ifos.path.exists(self.__SHANGHAIRANKING_ROOT):print('[info]: 文件创建成功')else:print('[error]: 文件创建失败')exit(1)self.workBook=xlsxwriter.Workbook(f'{self.__SHANGHAIRANKING_ROOT}/bcur_{year}.xlsx')print('[info]: workbook initialized...')def__closeWorkBook(self):self.workBook.close()print('[info]: workbook closed...')#init sheetdef__initSheet(self):ifself.workBook==0ornotisinstance(self.workBook,xlsxwriter.Workbook):print("[error]: init workbook first")self.close()exit(1)self.sheet=self.workBook.add_worksheet()#initial the table headself.sheet.write(0,0,'排名')self.sheet.write(0,1,'学校名称')self.sheet.write(0,2,'英文名称')self.sheet.write(0,3,'标签')self.sheet.write(0,4,'省市')self.sheet.write(0,5,'类型')self.sheet.write(0,6,'总分')foriinrange(len(self.indicator_list)):self.sheet.write(0,7+i,self.indicator_list[i])print('[info]: sheet initialized...')defrecording(self,year):offset=1url=f'https://www.shanghairanking.cn/rankings/bcur/{year}'self._driver.get(url)time.sleep(2)self.__initIndicatorList()self.__initWorkBook(year)self.__initSheet()whileTrue:table=self._driver.find_element_by_xpath('//*[@id="content-box"]/div[2]/table/tbody')trs=table.find_elements_by_tag_name('tr')curline=offsetfortrintrs:tds=tr.find_elements_by_tag_name('td')self.sheet.write(curline,0,tds[0].text)self.sheet.write(curline,1,tds[1].find_element_by_class_name('name-cn').text)self.sheet.write(curline,2,tds[1].find_element_by_class_name('name-en').text)try:tags=tds[1].find_element_by_class_name('tags')self.sheet.write(curline,3,tags.text)exceptexceptions.NoSuchElementException:self.sheet.write(curline,3,'')self.sheet.write(curline,4,tds[2].text)self.sheet.write(curline,5,tds[3].text)self.sheet.write(curline,6,tds[4].text)curline=curline+1forindexinrange(self.more_indicator_cnt):curline=offsetbutt=self._driver.find_element_by_xpath('//*[@id="content-box"]/div[2]/table/thead/tr/th[6]/div/div[1]/div[1]/input')self._driver.execute_script('arguments[0].click();',butt)menu=self._driver.find_element_by_xpath('//*[@id="content-box"]/div[2]/table/thead/tr/th[6]/div/div[1]/div[2]/ul')\.find_elements_by_css_selector('li')self._driver.execute_script('arguments[0].click();',menu[index])time.sleep(0.5)fortrintrs:self.sheet.write(curline,7+index,tr.find_elements_by_tag_name('td')[-1].text)curline=curline+1offset=curlinetry:self._driver.find_element_by_class_name('ant-pagination-disabled.ant-pagination-next')print('[processing:{}finished...]'.format(offset-1))print(f'[info]: shanghairanking of{year}is finished...')self.__closeWorkBook()returnexceptexceptions.NoSuchElementException:nextButt=self._driver.find_element_by_class_name('ant-pagination-next')self._driver.execute_script('arguments[0].click();',nextButt)print('[processing:{}finished...]'.format(offset-1))time.sleep(0.5)exceptBaseExceptionase:self.__closeWorkBook()raise(e)defclose(self):self._driver.close()if__name__=='__main__':shr=ShangHaiRanking()try:foriinrange(2018,2021):shr.recording(i)exceptBaseExceptionase:print('[error]: crashed, details are blow')print(traceback.format_exc(),end='')finally:shr.close()If using CrawlerRocket, things get much easier!todoThe scheduler :alarm_clock: :hourglass_flowing_sand: