tools/dbcore.py
def getConnection():
Crawler/ZojCrawler/ZojScanner.py
class ZojScanner:
    s = requests.session()

    scan_url = 'http://acm.zju.edu.cn/onlinejudge/showRuns.do?contestId=1&search=true&firstId=-1&lastId=-1&problemCode=&handle={}&idStart=&idEnd='

    def Analyse(self, html):

        soup = BeautifulSoup(html, 'html5lib')
        L = list()

        for i in range(2, 30):

            tr = soup.select_one('#SubmissionSearchForm > table > tbody > tr:nth-of-type({})'.format(i))
            if tr is None: break

            dt = dict()
            dt['originOJ'] = 'ZOJ'

            titles = ['realrunid', 'realsubmittime', 'status', 'originProb', 'language'
                , 'runtime', 'runmemory', 'nickname']

            for con in tr.contents:
                try:
                    dt[titles[0]] = con.text.strip()
                    titles = titles[1:]
                except Exception:
                    pass

            L.append(dt)

        return L

    def Scanner(self):

        L = list()

        for x in Zoj_User:
            url = self.scan_url.format(x['handle'])
            r = self.s.get(url, timeout=5)

            '''
            f = open('/tmp/R2.html','w')
            f.write(r.text)
            f.close()
            '''

            tl = self.Analyse(r.text)

            L += tl

        return L
tools/dbtools.py
def getUpdateSQL(table, data, clause):
    part1 = ''

    for key in data:

        if data[key] is None:
            continue

        if len(part1) != 0:
            part1 += ' , '

        if (isinstance(data[key], str)):
            part1 = part1 + key + ' = "' + str(data[key]) + '"'
        else:
            part1 = part1 + key + ' = ' + str(data[key])

    sql = 'UPDATE {} SET {} WHERE ( {} )'.format(table, part1, clause)

    return sql
Config/FilePathConfig.py
SID_DATA_FILE = '/home/ckboss/Desktop/Development/PKL/'
Crawler/HduCrawler/HduScanner.py
class HduScanner:
    s = requests.session()
    scan_url = 'http://acm.hdu.edu.cn/status.php?first=&pid=&user={}&lang=0&status=0'

    def Analyse(self, html):
        '''
        f = open('/tmp/status.html','w')
        f.write(html)
        f.close()
        '''
        soup = BeautifulSoup(html, 'html5lib')

        # print('-'*30)

        L = list()

        for i in range(2, 20):
            td = soup.select('#fixed_table > table > tbody > tr:nth-of-type({})'.format(i))
            if len(td) == 0: break

            dt = dict()
            dt['originOJ'] = 'HDU'
            titles = ['realrunid', 'realsubmittime', 'status', 'originProb', 'runtime',
                      'runmemory', 'codelenth', 'language', 'nickname']
            for con in td[0].contents:
                dt[titles[0]] = con.text
                if titles[0] == 'codelenth':
                    dt[titles[0]] = dt[titles[0]][:-1]
                titles = titles[1:]

            L.append(dt)

        return L

    def Scanner(self):

        L = list()

        for x in HduUser:
            url = self.scan_url.format(x.get('username'))
            r = self.s.get(url, timeout=5)
            r.encoding = 'gb2312'
            tL = self.Analyse(r.text)
            L += tL

        return L

    def UpdateToDB(self):
        pass
Crawler/PkuCrawler/PkuScanner.py
class PkuScanner:
    s = requests.session()

    scan_url = 'http://poj.org/status?problem_id=&user_id={}&result=&language='

    def Analyse(self, html):

        soup = BeautifulSoup(html, 'html5lib')

        L = list()

        for i in range(2, 30):

            td = soup.select_one('body > table.a > tbody > tr:nth-of-type({})'.format(i))

            if td is None: break

            dt = dict()
            dt['originOJ'] = 'PKU'

            titles = ['realrunid', 'nickname', 'originProb', 'status', 'runmemory',
                      'runtime', 'language', 'codelenth', 'realsubmittime']

            for con in td.contents:

                dt[titles[0]] = con.text
                if titles[0] == 'codelenth':
                    dt[titles[0]] = dt[titles[0]][:-1]
                titles = titles[1:]

            L.append(dt)

        return L

    def Scanner(self):

        L = list()

        for x in Pku_User:
            url = self.scan_url.format(x['user_id1'])
            r = self.s.get(url, timeout=5)

            '''
            f = open('/tmp/r1.text','w')
            f.write(r.text)
            f.close()
            '''

            tL = self.Analyse(r.text)

            L += tL

        return L
tools/dbtools.py
def ExeSQL(sql):
    try :
        conn = ConnPool.connect()
        cur = conn.cursor()
        cur.execute(sql)
        cur.close()
        conn.close()
    except Exception as e :
        print('sql exe Error {}'.format(str(e)))
        print('sql: ',sql)
        return False
    return True
Crawler/HustCrawler/HustScanner.py
class HustScanner:

    s = requests.session()
    scan_url = 'http://acm.hust.edu.cn/vjudge/problem/fetchStatus.action'

    dt = {
        'draw': '1',
        'columns[0][data]': '0',
        'columns[0][name]': '',
        'columns[0][searchable]': 'true',
        'columns[0][orderable]': 'false',
        'columns[0][search][value]': '',
        'columns[0][search][regex]': 'false',
        'order[0][column]': '0',
        'order[0][dir]': 'desc',
        'start': '0',
        'length': '20',
        'search[value]': '',
        'search[regex]': 'false',
        'un': '',
        'OJId': 'All',
        'probNum': '',
        'res': '0',
        'orderBy': 'run_id',
        'language': '',
    }

    def CheckLanguage(self,lang):

        lang=lang.lower()

        cpp = re.compile(r'.*c\+\+.*')
        gcc = re.compile(r'.*g\+\+.*')
        c = re.compile(r'.*c.*')
        java = re.compile(r'.*java.*')

        m1 = cpp.match(lang)
        m2 = gcc.match(lang)
        m3 = c.match(lang)
        m4 = java.match(lang)

        if m1 is not None or m2 is not None :
            return 'C++'
        elif m3 is not None :
            return 'C'
        elif m4 is not None:
            return 'Java'

    def Analyse(self,d):

        L = list()
        for i in range(0,20):
            try :
                tl = d['data'][i]
            except Exception as e :
                break
            item = dict()
            item['nickname'] = tl[1]
            item['realrunid'] = tl[0]
            item['originOJ'] = tl[11]
            item['originProb'] = tl[12]
            item['runmemory'] = tl[4]
            item['runtime'] = tl[5]
            item['language'] = self.CheckLanguage(tl[6])
            item['status'] = tl[3]
            L.append(item)
            self.CheckLanguage(item['language'])

        return L

    def Scanner(self):

        L = list()

        for x in HustUser :

            postdata = self.dt
            postdata['un'] = x['vj_username']
            r = self.s.post(url=self.scan_url,data=postdata)
            d = json.loads(r.text)
            tL = self.Analyse(d)

            L += tL

        return L
Crawler/BzojCrawler/BzojScanner.py
class BzojScanner:
    s = requests.session()
    scan_url = 'http://www.lydsy.com/JudgeOnline/status.php?problem_id=&user_id={}&language=-1&jresult=-1'

    def Analyse(self, html):

        soup = BeautifulSoup(html, 'html5lib')

        L = list()

        for i in range(2, 30):

            html = soup.select_one(
                'body > center:nth-of-type(1) > table:nth-of-type(2) > tbody > tr:nth-of-type({})'.format(i))
            if html is None: break

            dt = dict()
            dt['originOJ'] = 'BZOJ'
            Term = ['', 'realrunid', 'nickname', 'originProb', 'status', 'runmemory', 'runtime', 'language',
                    'codelenth', 'timesubmit']
            for j in range(1, 10):
                html = soup.select_one(
                    'body > center:nth-of-type(1) > table:nth-of-type(2) > tbody > tr:nth-of-type({}) > td:nth-of-type({})'.format(
                        i, j))
                dt[Term[j]] = html.text
                if Term[j] == 'codelenth':
                    dt[Term[j]] = str(dt[Term[j]]).replace(' B', '')

            L.append(dt)

        return L

    def Scanner(self):

        L = list()

        Users = list()
        Users += BzojUser
        Users += BzojVIPUser

        for x in Users:
            url = self.scan_url.format(x['user_id'])
            r = self.s.get(url, timeout=5)
            r.encoding = 'utf-8'

            html = r.text
            tL = self.Analyse(html)

            L += tL

        return L
tools/dbtools.py
def FetchOne(sql):

    conn = ConnPool.connect()
    cur = conn.cursor()
    cur.execute(sql)
    rs = cur.fetchone()
    cur.close()
    conn.close()

    return rs
Crawler/BnuVJCrawler/BnuVJScanner.py
class BnuVJScanner:

    s = requests.session()

    scan_url = 'https://www.bnuoj.com/v3/ajax/status_data.php?sEcho=6&iColumns=10&sColumns=&iDisplayStart=0&iDisplayLength=20&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&mDataProp_4=4&mDataProp_5=5&mDataProp_6=6&mDataProp_7=7&mDataProp_8=8&mDataProp_9=9&sSearch=&bRegex=false&sSearch_0={username}&bRegex_0=false&bSearchable_0=true&sSearch_1=&bRegex_1=false&bSearchable_1=true&sSearch_2=&bRegex_2=false&bSearchable_2=true&sSearch_3=&bRegex_3=false&bSearchable_3=true&sSearch_4=&bRegex_4=false&bSearchable_4=true&sSearch_5=&bRegex_5=false&bSearchable_5=true&sSearch_6=&bRegex_6=false&bSearchable_6=true&sSearch_7=&bRegex_7=false&bSearchable_7=true&sSearch_8=&bRegex_8=false&bSearchable_8=true&sSearch_9=&bRegex_9=false&bSearchable_9=true&iSortCol_0=1&sSortDir_0=desc&iSortingCols=1&bSortable_0=false&bSortable_1=false&bSortable_2=false&bSortable_3=false&bSortable_4=false&bSortable_5=false&bSortable_6=false&bSortable_7=false&bSortable_8=false&bSortable_9=false&_=1458028215625'

    def languageChange(self,lang):
        if lang == 'GNU C':
            return 'C'
        elif lang == 'GNU C++':
            return 'C++'
        else :
            return 'Java'

    def getOriginOJandProb(self,vpid):

        sql = getQueryDetailSQL('problem','originOJ,originProb',' virtualProb="{}" and virtualOJ="{}"'.format(vpid,'BNUVJ'),' pid ')

        #print('sql: ',sql)

        conn = ConnPool.connect()
        cur = conn.cursor()
        cur.execute(sql)
        rt = cur.fetchone()
        cur.close()
        conn.close()

        return rt


    def Analyse(self,html):

        jsondata = json.loads(html)

        n = len(jsondata['aaData'])
        L = list()


        for i in range(0,n) :

            dt = dict()
            dt['virtualOJ'] = 'BNUVJ'

            Title = ['nickname','realrunid','virtualProb','status','language',
                     'runtime','runmemory','codelength','realsubmittime']

            #print(jsondata['aaData'])
            for j in range(0,9):
                dt[Title[j]] = jsondata['aaData'][i][j]

            rt = self.getOriginOJandProb(dt['virtualProb'])

            if rt is not None :
                dt['originOJ'] = rt[0]
                dt['originProb'] = rt[1]

            dt['language'] = self.languageChange(dt['language'])

            L.append(dt)

        return L

    def Scanner(self):

        L = list()

        for x in BnuVJUser:
            url = self.scan_url.format(username=x['username'])
            r = self.s.get(url,timeout=5)

            html = r.text

            #print(html)

            tL = self.Analyse(html)

            L+=tL

        return L
tools/dbtools.py
def FetchAll(sql):

    conn = ConnPool.connect()
    cur = conn.cursor()
    cur.execute(sql)
    rs = cur.fetchall()
    cur.close()
    conn.close()

    return rs


Question: import time
import os
import pickle
import traceback
import threading
from Crawler.HduCrawler.HduScanner import HduScanner
from Crawler.PkuCrawler.PkuScanner import PkuScanner
from Crawler.ZojCrawler.ZojScanner import ZojScanner
from Crawler.BzojCrawler.BzojScanner import BzojScanner
from Crawler.BnuVJCrawler.BnuVJScanner import BnuVJScanner
from Crawler.HustCrawler.HustScanner import HustScanner
from tools.dbtools import getUpdateSQL,FetchOne,FetchAll,ExeSQL
from tools.dbcore import ConnPool
from Config.FilePathConfig import SID_DATA_FILE




LL = list()
lock = threading.Lock()

class ScannerThread(threading.Thread) :
    ojname = ''
    l = list()
    def __init__(self,ojname):
        threading.Thread.__init__(self)
        self.ojname = str(ojname).lower()

    def run(self):

        if self.ojname=='hdu':
            hs = HduScanner()
            l = hs.Scanner()
        elif self.ojname=='pku':
            ps = PkuScanner()
            l = ps.Scanner()
        elif self.ojname=='zoj':
            zs = ZojScanner()
            l = zs.Scanner()
        elif self.ojname=='bnuoj':
            bs = BnuVJScanner()
            l = bs.Scanner()
        elif self.ojname=='bzoj':
            bz = BzojScanner()
            l = bz.Scanner()
        elif self.ojname=='hust':
            hus = HustScanner()
            l = hus.Scanner()
        else:
            l = list()

        if lock.acquire(10):
            global LL
            LL += l
            lock.release()

        print(self.ojname,'thread catch:',len(l),'results')


class MainScanner():
    TF = SID_DATA_FILE
    ojs = ['hdu','pku','zoj','bzoj','bnuoj','hust']

    def Scanner(self):

        while True:
            try :
                self.Doit()
            except Exception:
                print('Some Exception in Scanner')
                traceback.print_exc()

            time.sleep(5)

    def initT(self,t):
        t.setDaemon(True)
        t.start()
        t.join(15)

    def GaoIt(self):
        global LL
        LL = list()
        tl = [ ScannerThread(self.ojs[i]) for i in range(0,len(self.ojs)) ]
        for t in tl : self.initT(t)
        return LL

    def Doit(self):

        files = os.listdir(self.TF)
        cont = False
        for file in files :
            if file.endswith('.pkl'):
                cont = True
                break
        if cont==False : return

        print('-> Before Scanner Stauts: ')
        #L = self.FindAndUpdate()
        L = self.GaoIt()
        print('-> End Scanner Stauts: ')

        '''
        print('Len L: ',len(L))
        for li in L : print('----> ',li)
        '''
        for file in files:
            if file.endswith('.pkl'):

                S = pickle.load(open(self.TF + file, 'rb'))
                # print('S:',S)

                for x in L:
                    try :
                        ret = self.CheckIt(S, x)
                    except Exception :
                        print('In check Exception: ')
                        print('S: ',S)
                        print('x: ',x)
                        traceback.print_exc()
                    #print('ret ---> ',ret)
                    if ret is None:
                        continue
                    else:
                        print('here is ret :', ret)
                        # update status
                        clause = 'sid = {}'.format(S['sid'])
                        sql = getUpdateSQL('status', ret, clause)
                        # print('update status sql: ',sql)

                        S['status'] = ret['status']

Answer:                         ExeSQL(sql)