petrel-sdk/petrel-oss-python-sdk/examples/pytorch/MyTest.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division
import threading
import logging

from MyDataset import MyDataset

import ceph

class testThread(threading.Thread):
	def __init__(self, threadid):
		threading.Thread.__init__(self)
		self.threadid = threadid

	def run(self):
		self.do_tasks()

	def do_tasks(self):
		face_dataset = MyDataset(csv_file='faces/face_landmarks.csv',root_dir='s3://yijianliang.test/train-copy0/')

		for i in range(len(face_dataset)):
			sample = face_dataset[i]
			#print(i, sample['image'].shape, sample['landmarks'].shape)
			logging.info('{0} {1} {2}'.format(i, sample['image'].shape, sample['landmarks'].shape))

'''
if __name__ == '__main__':
	logging.basicConfig(level=logging.INFO,
		format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
		datefmt='[%Y-%m_%d %H:%M:%S]',
		filename='log/MyTest.log',
		filemode='a')
	threads = []
	for i in range(0,1):
		threads.append(testThread(threadid=i))
	for thread in threads:
		thread.start()
	for thread in threads:
		thread.join()
'''

if __name__ == '__main__':
	logging.basicConfig(level=logging.INFO,
		format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
		datefmt='[%Y-%m_%d %H:%M:%S]')
		#filename='log/MyTest.log',
		#filemode='a')

	face_dataset = MyDataset(csv_file='faces/face_landmarks.csv',root_dir='s3://yijianliang.ssd.qos/train')
	for i in range(len(face_dataset)):
		sample, string_data = face_dataset[i]
		#print(i, sample['image'].shape, sample['landmarks'].shape)
		object_name = str(i)
		if sample and string_data:
			#s3client = ceph.S3Client()
			s3client = ceph.S3Client(access_key = "DWD2LKXJHJLGYKRDED7T", secret_key = "tzJ2a0g26deZZux3bLOd29YV9zJlaLM400Fu5tdn")
			ret = s3client.save_from_string('s3://sensestudytest/save_from_string/', object_name, string_data)
			if ret:
				logging.info('Save {0}: {1} bytes'.format(object_name, ret))
			logging.info('{0} {1} {2}'.format(i, sample['image'].shape, sample['landmarks'].shape))