#NEWLINE# def update_twitter( ) : #NEWLINE# #INDENT# oauth = OAuth1 ( os . environ [ 'TWITTER_CONSUMER_KEY' ] , os . environ [ 'TWITTER_CONSUMER_SECRET' ] , os . environ [ 'TWITTER_ACCESS_TOKEN' ] , os . environ [ 'TWITTER_ACCESS_TOKEN_SECRET' ] ) #NEWLINE# elsewhere = db . all ( "SELECT user_id FROM ELSEWHERE WHERE platform='twitter' ORDER BY id;" ) #NEWLINE# url = 'https://api.twitter.com/1.1/users/lookup.json' #NEWLINE# while elsewhere : #NEWLINE# #INDENT# batch = elsewhere [ : 100 ] #NEWLINE# elsewhere = elsewhere [ 100 : ] #NEWLINE# user_ids = ',' . join ( [ str ( user_id ) for user_id in batch ] ) #NEWLINE# response = requests . post ( url , data = { #NEWLINE# #INDENT# 'user_id' : user_ids , #NEWLINE# #UNINDENT# } , auth = oauth ) #NEWLINE# nremaining = int ( response . headers [ 'X-RATE-LIMIT-REMAINING' ] ) #NEWLINE# reset = int ( response . headers [ 'X-RATE-LIMIT-RESET' ] ) #NEWLINE# print ( nremaining , reset , time . time ( ) ) #NEWLINE# if ( response . status_code != 200 ) : #NEWLINE# #INDENT# print ( response . status_code , response . text ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# users = response . json ( ) #NEWLINE# with db . get_cursor ( ) as c : #NEWLINE# #INDENT# for user_info in users : #NEWLINE# #INDENT# for ( k , v ) in user_info . items ( ) : #NEWLINE# #INDENT# user_info [ k ] = unicode ( v ) #NEWLINE# #UNINDENT# user_id = user_info [ 'id' ] #NEWLINE# c . one ( "\n                        UPDATE elsewhere\n                        SET user_info=%s\n                        WHERE user_id=%s\n                        AND platform='twitter'\n                        RETURNING id\n                    " , ( user_info , user_id ) ) #NEWLINE# print ( 'updated {} ({})' . format ( user_info [ 'screen_name' ] , user_id ) ) #NEWLINE# #UNINDENT# existing = set ( ( u [ 'id' ] for u in users ) ) #NEWLINE# deleted = ( existing - set ( batch ) ) #NEWLINE# for user_id in deleted : #NEWLINE# #INDENT# c . one ( "\n                        UPDATE elsewhere\n                        SET user_info=NULL\n                        WHERE user_id=%s\n                        AND platform='twitter'\n                        RETURNING id\n                    " , ( user_id , ) ) #NEWLINE# print ( 'orphan found: {}' . format ( user_id ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# sleep_for = 5 #NEWLINE# if ( nremaining == 0 ) : #NEWLINE# #INDENT# sleep_for = ( reset - time . time ( ) ) #NEWLINE# sleep_for += 10 #NEWLINE# #UNINDENT# time . sleep ( sleep_for )