Spaces:
Runtime error
Runtime error
File size: 1,809 Bytes
485f76b f1ab0d5 485f76b ae7097b 485f76b f1ab0d5 7c115c7 74a29fd 7c115c7 f1ab0d5 74a29fd 485f76b dd9165a 485f76b dd9165a 485f76b 60ec487 485f76b f1ab0d5 485f76b dd9165a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
#!/usr/bin/env python3
import csv
import concurrent.futures
import requests
from progress.bar import ChargingBar
from entity import Entity
from common import defaults,mkdir
import screenshot
import web
PARALLEL = 20
def query_vendor_site(e: Entity):
fn = web.get_cert(e)
lfn = web.get_logos(e)
sfn = screenshot.sc_entity(e)
return (fn, lfn, sfn)
def from_csv(fn: str, n_workers = PARALLEL):
with open(fn, newline='') as csvfile:
reader = csv.DictReader(csvfile)
with concurrent.futures.ThreadPoolExecutor(max_workers = n_workers) as executor:
futures = {executor.submit(query_vendor_site, e): e for e in [Entity.from_dict(d) for d in reader]}
bar = ChargingBar('Processing', max=len(futures))
for f in concurrent.futures.as_completed(futures):
url = futures[f]
try:
(cert, logos) = f.result()
except Exception as exc:
print('%r generated an exception: %s' % (url, exc))
raise
else:
print(cert, logos)
bar.next()
bar.finish()
#query_vendor_site(Entity.from_dict({'url':'http://www.bancoprovincia.com.ar', 'bco':'debug'}))
#exit()
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='extract certificates and screenshots websites')
parser.add_argument('--csv', metavar='csv', type=str,
default=defaults.MAIN_CSV_PATH,
help='main database')
parser.add_argument('--parallel', metavar='parallel', type=int,
default=PARALLEL,
help='number of concurrent jobs')
args = parser.parse_args()
from_csv(args.csv)
|