File size: 1,244 Bytes
2abfccb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
# -*- coding: utf-8 -*-
from petrel_client.client import Client
from multiprocessing import Process
import logging
import random
LOG = logging.getLogger('petrel_client.test')
def f(conf_path, repeat):
client = Client(conf_path)
total_bytes = 0
for itr in range(repeat):
urls = [
'cluster1:s3://my-bucket/object.1', # 从 cluster1 中读取
'cluster2:s3://my-bucket/object.2', # 从 cluster2 中读取
's3://my-bucket/object.3', # 若不指定 cluster，则从配置文件中指定的 default_cluster 中读取
'file://tmp/xxx', # 从 DFS 中读取
'/tmp/xxx', # 若不包含 's3:' 或 'file:'，从 DFS 中读取
]
url = random.choice(urls)
body = client.get(url)
if not body:
LOG.warn('can not get content from %s', url)
else:
total_bytes += len(body)
LOG.debug('total_bytes: %s', total_bytes)
conf_path = '~/petreloss.conf'
repeat = 5000
parallelism = 4
process_list = [Process(target=f, args=(conf_path, repeat))
for _ in range(parallelism)]
[p.start() for p in process_list]
[p.join() for p in process_list]
|