Weiyun1025's picture
Upload folder using huggingface_hub
2abfccb verified
# -*- coding: utf-8 -*-
from __future__ import print_function, division
import threading
import logging
from MyDataset import MyDataset
import ceph
class testThread(threading.Thread):
def __init__(self, threadid):
threading.Thread.__init__(self)
self.threadid = threadid
def run(self):
self.do_tasks()
def do_tasks(self):
face_dataset = MyDataset(csv_file='faces/face_landmarks.csv',root_dir='s3://yijianliang.test/train-copy0/')
for i in range(len(face_dataset)):
sample = face_dataset[i]
#print(i, sample['image'].shape, sample['landmarks'].shape)
logging.info('{0} {1} {2}'.format(i, sample['image'].shape, sample['landmarks'].shape))
'''
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
datefmt='[%Y-%m_%d %H:%M:%S]',
filename='log/MyTest.log',
filemode='a')
threads = []
for i in range(0,1):
threads.append(testThread(threadid=i))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
'''
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
datefmt='[%Y-%m_%d %H:%M:%S]')
#filename='log/MyTest.log',
#filemode='a')
face_dataset = MyDataset(csv_file='faces/face_landmarks.csv',root_dir='s3://yijianliang.ssd.qos/train')
for i in range(len(face_dataset)):
sample, string_data = face_dataset[i]
#print(i, sample['image'].shape, sample['landmarks'].shape)
object_name = str(i)
if sample and string_data:
#s3client = ceph.S3Client()
s3client = ceph.S3Client(access_key = "DWD2LKXJHJLGYKRDED7T", secret_key = "tzJ2a0g26deZZux3bLOd29YV9zJlaLM400Fu5tdn")
ret = s3client.save_from_string('s3://sensestudytest/save_from_string/', object_name, string_data)
if ret:
logging.info('Save {0}: {1} bytes'.format(object_name, ret))
logging.info('{0} {1} {2}'.format(i, sample['image'].shape, sample['landmarks'].shape))