Package: py-collector
Py CollectorPy Collector is a simple, reliable, DB agnostic framework for consistently collecting data from
any source.It utilizes two main components, theCollectorand theScheduler.Checkout the imports to run the exampleshereCode ExamplesDataCodeCollect Weather Data into SQL AlchemyclassWeather(Collector):start_time=datetime.now()#to start immediatlyscheduler=Scheduler(days=1/24,count=1,separator=1,start_time=start_time)defupload(self):''' Runs on schedule, and will only run if is_newreturns true'''r=requests.get('https://api.weather.gov/gridpoints/FWD/59,23/forecast')data=r.json()['properties']['periods']points=[]foriindata:data_point=WeatherDataPoint(start_date=datetime.fromisoformat(i['startTime']),end_date=datetime.fromisoformat(i['endTime']),temp=i['temperature'],windspeed=i['windSpeed'])points.append(data_point)session.add_all(points)session.commit()defis_new(self):'''Evaluates if the data should be uploaded,if it only returns True, then it will just uploadon schedule.'''returnTrueCollect Energy Data into a CSV every minuteclassEnergy(Collector):start_time=datetime.now()#to start immediatlyscheduler=Scheduler(minutes=1,#every minutecount=2,#try 3 timesseparator=2,#two seconds between triesstart_time=start_time)first_run=Truelast_update=Nonedefupload(self):''' Runs on schedule, and will only run if is_newreturns true'''df=pd.read_html(self.get_site.text)[0]title='ercot_dam_clearing_'+self.last_update.strftime('%m_%d_%Y')+'.csv'file=open(title,'w')df.to_csv(file)defis_new(self):'''Evaluates if the data should be uploaded,if it only returns True, then it will just uploadon schedule.'''ifself.first_run:#first run load whatever is thereself.first_run=Falseself.last_update=self.get_last_changed()returnTrueelse:#if it has changed since we last updated, downloadlast_changed=self.get_last_changed()ifself.last_update<last_changed:self.last_update=last_changedreturnTrueelse:returnFalsedefget_site(self):returnrequests.get('http://www.ercot.com/content/cdr/html/actual_loads_of_forecast_zones')defsoup(self):r=self.get_site()returnBeautifulSoup(r.text,'html.parser')defget_last_changed(self):soup=self.soup()last_change=soup.find('div',attrs={'class':'schedTime rightAlign'})last_change=last_change.text.split('Time:')[1].lstrip()returndatetime.strptime(last_change,'%b%d, %Y %H:%M')Collect TikTok Data into a MongoDB every dayclassTikTokUser(MongoModel):username=fields.CharField()followers=fields.CharField()likes=fields.CharField()following=fields.CharField()classMeta:write_concern=WriteConcern(j=True)connection_alias='my-app'classTikTok(Collector):start_time=datetime.now()scheduler=Scheduler(days=1,#every datecount=1,#try 1 timesseparator=1,#not applicablestart_time=start_time)#start nowdefupload(self):''' Runs on schedule, and will only run if is_newreturns true'''data=self.user_stats('gordonramsayofficial')user=TikTokUser.from_document(data)user.save()defis_new(self):'''Evaluates if the data should be uploaded,if it only returns True, then it will just uploadon schedule.'''returnTruedefuser_stats(self,user='gordonramsayofficial'):r=self.user_raw(user)soup=BeautifulSoup(r.text,'html.parser')info=soup.find('h2',attrs={'class':'count-infos'})return{'following':info.find('strong',attrs={'title':'Following'}).text,'followers':info.find('strong',attrs={'title':'Followers'}).text,'likes':info.find('strong',attrs={'title':'Likes'}).text,'username':user}defuser_raw(self,user):headers={"authority":"m.tiktok.com","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0","method":"GET","scheme":"https","accept":"application/json, text/plain, */*","accept-encoding":'gzip, deflate, utf-8',"accept-language":"en-US,en;q=0.9","sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-site","sec-gpc":"1"}returnrequests.get(f'https://www.tiktok.com/@{user}?lang=en',headers=headers)