Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,28 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/pytorch/faces/face_landmarks.csv filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-cognito-identity.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-config.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-core.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-s3.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-cpp-sdk/pys3client.cpython-36-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-cpp-sdk/pys3client.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/examples/pytorch/faces/face_landmarks.csv filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/cache/mc/petrel_pymc.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/librgw/rados.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/librgw/rgw.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/s3cpp/libs/libaws-cpp-sdk-core.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/s3cpp/libs/libaws-cpp-sdk-s3.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/s3cpp/pys3client.cpython-36-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/s3cpp/pys3client.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/s3cpp/pys3client.cpython-37m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel-sdk/petrel-oss-python-sdk/petrel_client/ceph/s3cpp/pys3client.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/cache/mc/petrel_pymc.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/librgw/rados.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/librgw/rgw.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/s3cpp/libs/libaws-cpp-sdk-core.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/s3cpp/libs/libaws-cpp-sdk-s3.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/s3cpp/pys3client.cpython-36m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/s3cpp/pys3client.cpython-37m-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+petrel_client/ceph/s3cpp/pys3client.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,8 @@
+petrel_client/version.py
+.vscode
+**/__pycache__
+**/*.pyc
+**/*.egg-info
+venv
+dist
+build

.gitlab-ci.yml

@@ -0,0 +1,24 @@
+image: registry.sensetime.com/library/python:3.6
+
+
+before_script:
+  - python -V
+  - python -m venv venv
+  - source ./venv/bin/activate
+  - python setup.py sdist
+  - pip install dist/*
+
+stages:
+  - flake8
+  - unit_test
+
+flake8:
+  stage: flake8
+  script:
+    - pip install flake8
+    - flake8 --ignore E501 --exclude '.git,.tox,*.egg-info,venv,scripts,tests,examples' .  # todo remove scripts,tests,examples
+
+test:
+  stage: unit_test
+  script:
+    - python tests/run_test.py

INSTALL

@@ -0,0 +1,2 @@
+python setup.py install
+

README.md

@@ -0,0 +1,304 @@
+Petrel OSS SDK 2.0
+===
+
+注意：该版本SDK需要python3.6环境
+
+若之前安装过旧版本，请先运行
+
+```bash
+$ pip uninstall pycephs3client
+$ rm -rf ~/.local/lib/python3.6/site-packages/petrel_client
+```
+
+## 建议在安装之前先升级 pip
+
+```bash
+source /mnt/lustre/share/platform/env/<pat... or pt...> # 请根据实际情况确定是否需要 source
+python3 -m pip install --upgrade pip # 请根据实际情况确定是否需要 `sudo` 或添加 `--user` 参数
+```
+
+## 训练集群环境上安装
+
+```bash
+$ source /mnt/lustre/share/platform/env/<pat... or pt...>
+$ python setup.py sdist
+$ pip install --user dist/*
+```
+
+## 通过修改 PYTHONPATH 安装
+
+```bash
+$ source /mnt/lustre/share/platform/env/<pat... or pt...>
+
+# 安装SDK依赖
+$ python setup.py egg_info
+$ pip install -r *.egg-info/requires.txt
+
+# 将SDK编译到 ./build 目录
+$ python setup.py build
+
+# 修改 PYTHONPATH 环境变量
+$ export PYTHONPATH=<path_to_sdk>/build/lib:$PYTHONPATH
+```
+
+## venv环境上安装
+
+```bash
+$ python3 -m venv your_venv_name # 若已创建venv环境则无需执行
+$ source your_venv_name/bin/active
+$ python setup.py sdist
+$ pip install dist/*
+```
+
+## 系统环境上安装
+
+```bash
+$ python3 setup.py sdist
+$ python3 -m pip install dist/* # 请根据实际情况确定是否需要 `sudo` 或添加 `--user` 参数
+```
+
+## 使用
+
+SDK 提供 `get` 和 `put` 接口，使用方式为
+
+```python
+data = client.get(url)                      # 默认情况由配置文件决定是否使用 MC
+data = client.get(url, no_cache=True)       # 本次 get 直接从 ceph 读取
+data = client.get(url, update_cache=True)   # 本次 get 直接从 ceph 读取，并将数据缓存至 MC
+```
+
+```python
+client.put(url, data)                       # 默认 put 不会更新 MC
+client.put(url, data, update_cache=True)    # 本次 put 将数据存入 ceph 之后并更新 MC
+```
+
+``注意：``若配置文件中没有启用 `MC` ，则 `no_cache` 和 `update_cache` 参数将被忽略
+
+以下为使用 SDK 读取图片、进行图片处理后并保存图片的简单例子
+
+```python
+import cv2
+import numpy as np
+from os.path import splitext
+from petrel_client.client import Client
+
+conf_path = '~/petreloss.conf'
+client = Client(conf_path) # 若不指定 conf_path ，则从 '~/petreloss.conf' 读取配置文件
+img_url = 's3://bucket1/image.jpeg'
+img_gray_url = 's3://bucket1/image_gray.jpeg'
+img_ext = splitext(img_gray_url)[-1]
+
+# 图片读取
+img_bytes = client.get(img_url)
+assert(img_bytes is not None)
+img_mem_view = memoryview(img_bytes)
+img_array = np.frombuffer(img_mem_view, np.uint8)
+img = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
+
+# 图片处理
+img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+# 图片存储
+success, img_gray_array = cv2.imencode(img_ext, img_gray)
+assert(success)
+img_gray_bytes = img_gray_array.tostring()
+client.put(img_gray_url, img_gray_bytes)
+```
+
+配置文件请参考 [petreloss.conf](./conf/petreloss.conf)
+
+``请注意：配置文件中 `key = value` 的 key 前面不能有空格，否则该行视为上一行配置项 value 的一部分``
+
+使用样例请参考 [multi_cluster_test.py](./tests/multi_cluster_test.py)
+
+## `Tensor` 和 `Json` 数据保存与读取
+使用样例 [tensor_json_test.py](./tests/tensor_json_test.py)
+
+## 数据过大无法上传，则需要分片上传
+使用样例 [multipart_test.py](./tests/multipart_test.py)
+
+## 创建 Bucket
+```python
+client.create_bucket('s3://mybucket')
+```
+
+## 顺序的读取某个前缀的数据
+```python
+cluster = 'cluster1'
+files = client.get_file_iterator('cluster1:s3://lili1.test2/test3')
+for p, k in files
+    key = '{0}:s3://{1}'.format(cluster, p)
+    data = client.get(key)
+```
+## 使用 anonymous 账户访问数据
+若在配置文件中不设置 `access_key` 和 `secret_key`，将以 `anonymous` 账户访问数据。
+
+
+## McKeySizeExceed 错误
+
+默认情况下，`MC` 所支持 `key` 的最大长度为250个字节。如果路径过长，将会出现 `McKeySizeExceed` 错误。
+此时需要用户定义 `key` 的转换规则来避免该错误。
+
+``注意：``中文字符对应多个字节。
+
+例如：
+
+```python
+def trim_key(key):
+    if isinstance(key, str):
+        key = key.encode('utf-8')
+    else:
+        assert isinstance(key, bytes)
+
+    return key[-249:]
+
+client = Client('~/petreloss.conf', mc_key_cb=trim_key)
+```
+
+此外，可使用内置函数 `md5`、`sha256` 等，例如：
+
+```python
+client = Client('~/petreloss.conf', mc_key_cb='sha256')
+```
+
+或在配置文件中指定：
+
+```conf
+[mc]
+mc_key_cb = sha512
+```
+
+``请注意``
+
+- 用户需要保证转换规则结果的唯一性，内置转换函数也有可能发生哈希碰撞。
+- 如果 `key` 为 `str` 类型且其中出现中文字符，请务必用 `encode('utf-8')` 对其进行编码。
+
+
+## 使用伪客户端
+
+在对应客户端添加如下配置:
+
+```conf
+fake = True
+```
+
+配置文件请参考 [fake_client.conf](./conf/fake_client.conf)
+
+使用样例请参考 [fake_client_test.py](./tests/fake_client_test.py)
+
+## 强制更新MC
+
+使用 `get_and_update` 接口或在 `get` 中传入 `update_cache=True` 参数将直接从存储系统读取数据并更新MC。
+
+## IO 统计信息
+
+IO 统计信息可通过以下三种方式修改其`log`输出频度：
+- 由环境变量 `count_disp` 设置
+- 由配置文件 `count_disp` 设置 （若已设置环境变量，则该方式无效）
+- 调用 `client.set_count_disp(count_disp)` (该方式将覆盖上述两种方式），但限于`parrots`和`pytorch`的运行机制，在某些使用场景下可能无法有效修改。
+
+若 `count_disp` 为 `0` ，则将关闭 IO 统计信息打印。
+
+若需要在 `console` 中打印 IO 统计信息，则需要设置 `console_log_level` 为 `INFO` 或更低级别，且 `count_disp` 需大于 `0`。
+
+
+## DataLoader
+
+`SDK` 提供的 `DataLoader` 额外支持如下参数：
+
+- `prefetch_factor`，默认2。每个 `worker` 预读 `batch` 数目。
+- `persistent_workers`，默认 `False`。如果为 `True`，则每轮 `epoch` 迭代完毕后 `worker` 进程将不会关闭，下轮 `epoch` 将复用该 `worker` 进程。
+
+用例：
+
+```python
+from petrel_client.utils.data import DataLoader
+dataloader = DataLoader(dataset=xxx, ..., prefetch_factor=4, persistent_workers=True)
+```
+
+## SSL 验证
+
+使用 `https` 协议时默认不会对 `SSL` 进行验证。若需要开启验证，请在配置文件中进行如下设置
+```conf
+verify_ssl = True
+```
+
+## Presigned URL，生成签名链接
+
+```python
+presigned_url = client.generate_presigned_url(url, client_method ='get_object', expires_in=3600)
+```
+
+`client_method` 取值为 `get_object` (默认值) 或 `put_object`
+
+`expires_in` 单位为秒，默认值为 3600
+
+## Presigned POST，生成签名 POST
+
+```python
+presigned_post = client.generate_presigned_post(url, fields=None, conditions=None, expires_in=3600)
+```
+
+参数及返回值详见 [generate_presigned_post](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.generate_presigned_post)，其中参数 bucket 和 key 从 url 中提取。
+
+## 以流的形式读取数据
+```python
+stream = client.get(url, enable_stream=True)
+```
+返回的 `stream` 为 `StreamingBody`，使用方法详见
+https://botocore.amazonaws.com/v1/documentation/api/latest/reference/response.html
+
+## 判断对象是否存在
+```python
+exists = client.contains(url)
+```
+
+## 删除对象
+```python
+client.delete(url)
+```
+
+## 列出当前路径包含的对象或目录
+```python
+contents = client.list(url)
+for content in contents:
+    if content.endswith('/'):
+        print('directory:', content)
+    else:
+        print('object:', content)
+```
+
+## 判断目录是否存在
+```python
+client.isdir(url)
+```
+
+注意：`Ceph`中没有目录的概念，本函数返回`True`时代表存在以该`url`作为前缀的对象，其他情况返回`False`。
+
+
+## 使用 `/mnt/cache` 目录下的 `Python` 环境
+相对于 `/mnt/lustre` 目录，在 `/mnt/cache` 目录执行 `Python` 有一定的性能提升。
+使用方式如下:
+- `source` `/mnt/cache` 目录下的 `Python` 环境
+```bash
+### 例如 pt1.3v1
+source /mnt/cache/share/platform/env/pt1.3v1
+### 或 s0.3.3
+source /mnt/cache/share/spring/s0.3.3
+```
+
+- 检查 `Python` 路径是否正确
+```bash
+which python
+### 结果应为 /mnt/cache/...
+```
+
+- 设定 `PYTHONUSERBASE` 环境变量
+```bash
+export PYTHONUSERBASE=/mnt/cache/<username>/.local
+```
+
+- 重新安装相关依赖库（仅需首次使用时执行）
+```
+python -m pip install --user <packages>
+```

conf/ceph.conf

@@ -0,0 +1,24 @@
+[global]
+fsid = 85e75c85-ab98-426d-81cf-9daa534887f9
+mon_initial_members = SZ-OFFICE3-172-30-1-75
+mon_host = 172.30.1.75
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+
+public_network=172.30.1.0/20
+cluster_network=172.30.1.0/20
+
+mon_allow_pool_delete = True
+
+debug_mon = 20/20
+debug_client = 20/20
+
+rgw_enable_gc_threads = False
+rgw_enable_lc_threads = False
+rgw_enable_quota_threads = False
+rgw_run_sync_thread = False
+rgw enable ops log = False
+rgw enable usage log = False
+admin socket = ""

conf/fake_client.conf

@@ -0,0 +1,17 @@
+[DEFAULT]
+default_cluster = cluster1
+
+[dfs]
+fake = True
+enable_mc = True
+
+[mc]
+fake = True
+
+[cluster1]
+fake = True
+enable_mc = True
+
+[cluster2]
+fake = True
+enable_mc = False

conf/keyring

@@ -0,0 +1,2 @@
+[client.rgw.train]
+	key = AQBiiqVeaPzSFRAAT1Vc+z8wPI5BkCroB6W/jQ==

conf/petreloss.conf

@@ -0,0 +1,88 @@
+# 注释以 ’#‘ 或 ‘;’ 开头，单独占一行，不能和配置内容在同一行
+# `key = value` 的 key 前面不能有空格，否则该行视为上一行配置项 value 的一部分
+
+[DEFAULT]
+
+# 启用 Memcached, 默认 False
+# enable_mc = True
+
+# Memcached 相关配置，默认情况下无需设置
+# mc_server_list_path = /mnt/lustre/share/memcached_client/server_list.conf
+# mc_client_config_path = /mnt/lustre/share/memcached_client/client.conf
+
+# console log 级别，默认 WARNING, 选项有 CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET
+# 若需要在 console 输出 IO 统计信息，需要设置级别为 INFO
+# console_log_level = WARNING
+
+# file log 级别，默认 DEBUG, 选项有 CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET
+# file_log_level = DEBUG
+
+# log 文件路径，默认 无 ，即不输出 log 文件
+# 若已配置 log 文件路径，则训练运行之前需要确保 log_file_path 指定的目录已被创建
+# log_file_path = /mnt/lustre/<username>/petrel_log_dir
+
+# log 文件最大长度，默认 1GB
+# file_log_max_bytes = 1073741824
+
+# log 文件备份数目，默认 1
+# file_log_backup_count = 1
+
+# 每隔 count_disp 次 get 操作后，日志记录 IO 统计信息。默认值 5000
+# 如果 IO 统计信息输出过于频繁，可将该数值增大
+# 如果需要关闭 IO 统计信，可将该数值设置为 0
+# count_disp = 5000
+
+# 内存统计，默认关闭
+# enable_mem_trace = False
+
+# get 操作失败后，允许重试的次数，默认 10
+# get_retry_max = 10
+
+# 默认 cluster，即当访问 Ceph 没有指定 cluster 时，从 default_cluster 获取数据
+default_cluster = cluster1
+
+[mc]
+# 若访问的路径过长（超过250个字节），mc 将出现 McKeySizeExceed 错误。
+# 配置 mc_key_cb 可将传给 mc 的路径进行转换，可选的参数有:
+# blake2b, blake2s, md5, pbkdf2_hmac, sha1, sha224,
+# sha256, sha384, sha3_224, sha3_256, sha3_384,
+# sha3_512, sha512, shake_128, shake_256
+
+# mc_key_cb = sha256
+
+
+# 是否输出 mc debug log，默认 True
+# 注意最终是否输出到 console 和 file 分别还需要由 console_log_level 和 file_log_level 决定
+# debug_mc = True
+
+
+[dfs]
+enable_mc = True
+
+# 至少需要配置一个 cluster ，否则将出现 ConfigSectionNotFoundError
+[cluster1]
+# 对于每个集群的具体配置，如果没有指定，则以[DEFAULT]作为取值
+# 例如在此处设置 ‘enable_mc = False’ 将覆盖默认配置
+enable_mc = True
+
+# 启用 S3 boto，默认 True
+# boto = c++ 将启用 c++ 版本实现的 S3
+boto = True
+
+# 若不设置 access_key 和 secret_key，将以 anonymous 账户访问数据
+access_key = lili1
+secret_key = lili1
+
+host_base = http://127.0.0.1:7480
+
+# 若 boto = False ，则需要增加以下配置
+# conf = conf/ceph.conf
+# keyring = conf/keyring
+# name = client.rgw.train
+# cluster = ceph
+
+[cluster2]
+
+access_key = lili1
+secret_key = lili1
+host_base = http://127.0.0.1:7480

dev.sh

@@ -0,0 +1,4 @@
+python3 -m venv venv
+source venv/bin/activate
+pip install -U autopep8
+pip install -e .

examples/pytorch/MyDataset.py

@@ -0,0 +1,50 @@
+from __future__ import print_function, division
+import os
+import torch
+import pandas as pd
+from skimage import io, transform
+import matplotlib.pyplot as plt
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms, utils
+
+import cv2
+import numpy as np
+import ceph
+
+class MyDataset(Dataset):
+	def __init__(self, csv_file, root_dir):
+		"""
+		Args:
+			csv_file (string): Path to the csv file with annotations.
+			root_dir (string): Bucket with all the images, such as s3://faces/
+		"""
+		self.landmarks_frame = pd.read_csv(csv_file)
+		self.root_dir = root_dir
+		
+	def __len__(self):
+		return len(self.landmarks_frame)
+
+	def __getitem__(self, idx):
+		img_name = os.path.join(self.root_dir, self.landmarks_frame.iloc[idx, 0])
+		s3client = ceph.S3Client()	
+		value = s3client.get(img_name)
+		if not value:
+			"""
+			Picture doesn't exist in ceph, your code here to handle error
+			"""
+			return None
+		img_array = np.fromstring(value, np.uint8)
+		# load image
+		#img = cvb.img_from_bytes(value)
+
+		
+		string_data = img_array.tostring()
+		#print(string_data)
+		#print(value)
+		#image = cv2.imdecode(img_array, cv2.CV_LOAD_IMAGE_COLOR)
+
+		landmarks = self.landmarks_frame.iloc[idx, 1:].as_matrix()
+		landmarks = landmarks.astype('float').reshape(-1, 2)
+		sample = {'image': img_array, 'landmarks': landmarks}
+
+		return sample, string_data

examples/pytorch/MyTest.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, division
+import threading
+import logging
+
+from MyDataset import MyDataset
+
+import ceph
+
+class testThread(threading.Thread):
+	def __init__(self, threadid):
+		threading.Thread.__init__(self)
+		self.threadid = threadid
+
+	def run(self):
+		self.do_tasks()
+
+	def do_tasks(self):
+		face_dataset = MyDataset(csv_file='faces/face_landmarks.csv',root_dir='s3://yijianliang.test/train-copy0/')
+
+		for i in range(len(face_dataset)):
+			sample = face_dataset[i]
+			#print(i, sample['image'].shape, sample['landmarks'].shape)
+			logging.info('{0} {1} {2}'.format(i, sample['image'].shape, sample['landmarks'].shape))
+
+'''
+if __name__ == '__main__':
+	logging.basicConfig(level=logging.INFO,
+		format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
+		datefmt='[%Y-%m_%d %H:%M:%S]',
+		filename='log/MyTest.log',
+		filemode='a')
+	threads = []
+	for i in range(0,1):
+		threads.append(testThread(threadid=i))
+	for thread in threads:
+		thread.start()
+	for thread in threads:
+		thread.join()
+'''
+
+if __name__ == '__main__':
+	logging.basicConfig(level=logging.INFO,
+		format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
+		datefmt='[%Y-%m_%d %H:%M:%S]')
+		#filename='log/MyTest.log',
+		#filemode='a')
+
+	face_dataset = MyDataset(csv_file='faces/face_landmarks.csv',root_dir='s3://yijianliang.ssd.qos/train')
+	for i in range(len(face_dataset)):
+		sample, string_data = face_dataset[i]
+		#print(i, sample['image'].shape, sample['landmarks'].shape)
+		object_name = str(i)
+		if sample and string_data:
+			#s3client = ceph.S3Client()
+			s3client = ceph.S3Client(access_key = "DWD2LKXJHJLGYKRDED7T", secret_key = "tzJ2a0g26deZZux3bLOd29YV9zJlaLM400Fu5tdn")
+			ret = s3client.save_from_string('s3://sensestudytest/save_from_string/', object_name, string_data)
+			if ret:
+				logging.info('Save {0}: {1} bytes'.format(object_name, ret))
+			logging.info('{0} {1} {2}'.format(i, sample['image'].shape, sample['landmarks'].shape))

examples/pytorch/faces/face_landmarks.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82bd06902826e8a79658a5946187a4f441b9d21f8edceec4dfa8723ffcfcbad0
+size 63189398

examples/pytorch/log/MyTest.log

@@ -0,0 +1,3 @@
+[2018-04_18 13:45:16] MyTest.py[line:22] INFO 0 (250, 250, 3) (3, 2)
+[2018-04_18 13:45:16] MyTest.py[line:22] INFO 0 (250, 250, 3) (3, 2)
+[2018-04_18 13:45:16] MyTest.py[line:22] INFO 0 (250, 250, 3) (3, 2)

petrel-sdk-H.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4bd0ab2f7ca22aa16c83496358025e136ee2eea3fde591d0574c5b3a6f82d41
+size 21200532

petrel-sdk/petrel-oss-cpp-sdk/.gitignore

@@ -0,0 +1,5 @@
+build
+venv
+pys3client.cpp
+pys3client.cpython-36m-x86_64-linux-gnu.so
+pys3client.so

petrel-sdk/petrel-oss-cpp-sdk/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "aws-sdk-cpp"]
+	path = aws-sdk-cpp
+	url = http://gitlab.bj.sensetime.com/platform/StorageSystem/aws-sdk-cpp.git

petrel-sdk/petrel-oss-cpp-sdk/README.md

@@ -0,0 +1,6 @@
+# petrel-oss-cpp-sdk
+
+```bash
+scl enable devtoolset-8 -- bash
+python3 setup.py build_ext --inplace
+```

petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-cognito-identity.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce0525ae1cf99e3f99352f67cb1dbc5cc3f45e007474e71646e445aff283c167
+size 1203624

petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-config.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d91cc9c68c489ba0a16e63eddd8354fc9b8de53bb991d5f2e7a911fcc38f81a
+size 4489616

petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-core.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eed1dcb18c4481516fabbbe5d1fe11559af4a7efa196227f75da4f38e8428a1
+size 1465968

petrel-sdk/petrel-oss-cpp-sdk/libs/libaws-cpp-sdk-s3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bde1e1ba070640c53b51d885ca4ffd12fcedb031ca3a6231507f499c385d39ef
+size 4416312

petrel-sdk/petrel-oss-cpp-sdk/pys3client.cpython-36-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8484af53fea398ba08cd78502a0400b61aeae73315ffed5c2900d17abbc067d
+size 1109320

petrel-sdk/petrel-oss-cpp-sdk/pys3client.cpython-38-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:629201bfbedb0b1cddadbe6ec2f982fab941ee1cb04697d2407e596b2fda3cc5
+size 1145912

petrel-sdk/petrel-oss-cpp-sdk/pys3client.pyx

@@ -0,0 +1,91 @@
+# distutils: language = c++
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from s3client cimport S3Client, get_error_list
+from s3client cimport init_api as _init_api
+from s3client cimport shutdown_api as _shutdown_api
+
+
+ERROR_LIST = get_error_list()
+ERROR_MAP = {k:v.decode('utf-8') for k,v in ERROR_LIST}
+
+def init_api(log_level):
+    log_level = log_level.lower()
+    _init_api(log_level.encode('utf-8'))
+
+def shutdown_api():
+    _shutdown_api()
+
+class S3Error(Exception):
+    def __init__(self, error_name, error_message):
+        self.error_name = error_name
+        self.error_message = error_message
+
+cdef class PyS3Client:
+    cdef S3Client *client
+
+    def __cinit__(self, string ak, string sk, string endpoint, bint verify_ssl, bint enable_https, bint use_dual_stack, int threads_num):
+        self.client = new S3Client(ak, sk, endpoint, verify_ssl, enable_https, use_dual_stack, threads_num)
+    
+    def __dealloc__(self):
+        del self.client
+
+    def get_object(self, string bucket, string key, string range):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.get_object(bucket, key, error_type, error_message, result, range)
+        if ret == 0:
+            return result
+        else:
+            error_name = ERROR_MAP.get(error_type, 'Undefined')
+            raise S3Error(error_name, error_message)
+
+    def multipart_download_concurrency(self, string bucket, string key, string filename):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.multipart_download_concurrency(bucket, key, filename, error_type, error_message)
+        if ret == 0:
+            return ret
+        else:
+            error_name = ERROR_MAP.get(error_type, 'Undefined')
+            raise S3Error(error_name, error_message)
+
+    def put_object(self, string bucket, string key, string data):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.put_object(bucket, key, data, error_type, error_message)
+        if ret == 0:
+            return data.size()
+        else:
+            error_name = ERROR_MAP.get(error_type, 'Undefined')
+            raise S3Error(error_name, error_message)
+
+
+    def multipart_upload_concurrency(self, string bucket, string key, string filename):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.multipart_upload_concurrency(bucket, key, filename, error_type, error_message)
+        if ret == 0:
+            return ret
+        else:
+            error_name = ERROR_MAP.get(error_type, 'Undefined')
+            raise S3Error(error_name, error_message)
+
+    def delete(self, string bucket, string key):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.delete_obj(bucket, key, error_type, error_message)
+        return ret
+
+    def contains(self, string bucket, string key):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.contains(bucket, key, error_type, error_message)
+        return ret
+
+    def list(self, string bucket, string key):
+        cdef string error_message, result
+        cdef int error_type
+        ret = self.client.list(bucket, key, error_type, error_message)
+        return ret

petrel-sdk/petrel-oss-cpp-sdk/s3client.cpp

@@ -0,0 +1,323 @@
+#include <aws/core/Aws.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/Bucket.h>
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/ListObjectsRequest.h>
+#include <aws/s3/model/Object.h>
+#include <aws/s3/S3Errors.h>
+#include <string>
+#include <list>
+#include <fstream>
+#include "s3client.h"
+
+#include <aws/s3/model/CreateMultipartUploadRequest.h>
+#include <aws/s3/model/CompletedPart.h>
+#include <aws/s3/model/UploadPartRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h> 
+#include <aws/s3/model/CompletedMultipartUpload.h>
+#include <aws/s3/model/AbortMultipartUploadRequest.h>
+
+
+#include <iostream>
+#include <vector>
+
+// mutipart相关
+#include <aws/core/utils/threading/Executor.h>
+#include <aws/transfer/TransferManager.h>
+#include <aws/transfer/TransferHandle.h>
+#include <aws/core/utils/memory/AWSMemory.h>
+#include <aws/core/utils/memory/stl/AWSStreamFwd.h>
+#include <aws/core/utils/stream/PreallocatedStreamBuf.h>
+#include <aws/core/utils/StringUtils.h>
+
+#include <aws/s3/model/DeleteObjectRequest.h> 
+#include <aws/s3/model/HeadObjectRequest.h> 
+#include <aws/s3/model/ListObjectsV2Request.h> 
+
+#define ERROR_ITEM(error_name)                                       \
+    {                                                                \
+        static_cast<int>(Aws::S3::S3Errors::error_name), #error_name \
+    }
+
+static std::list<std::pair<int, std::string>> ERROR_LIST = {
+    ERROR_ITEM(INCOMPLETE_SIGNATURE),
+    ERROR_ITEM(INTERNAL_FAILURE),
+    ERROR_ITEM(INVALID_ACTION),
+    ERROR_ITEM(INVALID_CLIENT_TOKEN_ID),
+    ERROR_ITEM(INVALID_PARAMETER_COMBINATION),
+    ERROR_ITEM(INVALID_QUERY_PARAMETER),
+    ERROR_ITEM(INVALID_PARAMETER_VALUE),
+    ERROR_ITEM(MISSING_ACTION),
+    ERROR_ITEM(MISSING_AUTHENTICATION_TOKEN),
+    ERROR_ITEM(MISSING_PARAMETER),
+    ERROR_ITEM(OPT_IN_REQUIRED),
+    ERROR_ITEM(REQUEST_EXPIRED),
+    ERROR_ITEM(SERVICE_UNAVAILABLE),
+    ERROR_ITEM(THROTTLING),
+    ERROR_ITEM(VALIDATION),
+    ERROR_ITEM(ACCESS_DENIED),
+    ERROR_ITEM(RESOURCE_NOT_FOUND),
+    ERROR_ITEM(UNRECOGNIZED_CLIENT),
+    ERROR_ITEM(MALFORMED_QUERY_STRING),
+    ERROR_ITEM(SLOW_DOWN),
+    ERROR_ITEM(REQUEST_TIME_TOO_SKEWED),
+    ERROR_ITEM(INVALID_SIGNATURE),
+    ERROR_ITEM(SIGNATURE_DOES_NOT_MATCH),
+    ERROR_ITEM(INVALID_ACCESS_KEY_ID),
+    ERROR_ITEM(REQUEST_TIMEOUT),
+    ERROR_ITEM(NETWORK_CONNECTION),
+    ERROR_ITEM(UNKNOWN),
+    ERROR_ITEM(BUCKET_ALREADY_EXISTS),
+    ERROR_ITEM(BUCKET_ALREADY_OWNED_BY_YOU),
+    ERROR_ITEM(NO_SUCH_BUCKET),
+    ERROR_ITEM(NO_SUCH_KEY),
+    ERROR_ITEM(NO_SUCH_UPLOAD),
+    ERROR_ITEM(OBJECT_ALREADY_IN_ACTIVE_TIER),
+    ERROR_ITEM(OBJECT_NOT_IN_ACTIVE_TIER),
+};
+
+std::list<std::pair<int, std::string>> get_error_list()
+{
+    return ERROR_LIST;
+}
+static Aws::SDKOptions options;
+
+static std::unordered_map<std::string, Aws::Utils::Logging::LogLevel> log_level_map = {
+    {"off", Aws::Utils::Logging::LogLevel::Off},
+    {"fatal", Aws::Utils::Logging::LogLevel::Fatal},
+    {"error", Aws::Utils::Logging::LogLevel::Error},
+    {"warn", Aws::Utils::Logging::LogLevel::Warn},
+    {"info", Aws::Utils::Logging::LogLevel::Info},
+    {"debug", Aws::Utils::Logging::LogLevel::Debug},
+    {"trace", Aws::Utils::Logging::LogLevel::Trace},
+};
+
+void init_api(const std::string &level)
+{
+    auto itr = log_level_map.find(level);
+    if (itr != log_level_map.end())
+    {
+        options.loggingOptions.logLevel = itr->second;
+    }
+    Aws::InitAPI(options);
+}
+
+void shutdown_api()
+{
+    Aws::ShutdownAPI(options);
+}
+
+S3Client::S3Client(const std::string &ak, const std::string &sk, const std::string &endpoint, bool verify_ssl, bool enable_https, bool use_dual_stack, int threads_num)
+{
+    Aws::Client::ClientConfiguration config;
+    config.endpointOverride = endpoint.c_str();
+    config.verifySSL = verify_ssl;
+    if (enable_https)
+    {
+        config.scheme = Aws::Http::Scheme::HTTPS;
+    }
+    else
+    {
+        config.scheme = Aws::Http::Scheme::HTTP;
+    }
+
+    config.useDualStack = use_dual_stack;
+    Aws::Auth::AWSCredentials cred = Aws::Auth::AWSCredentials(ak.c_str(), sk.c_str());
+    // this->client = new Aws::S3::S3Client(cred, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false);
+    this->client = Aws::MakeShared<Aws::S3::S3Client>("S3Client", cred, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false);
+
+    // multipart使用
+    this->threads_num = threads_num;
+
+    return;
+}
+
+S3Client::~S3Client()
+{
+    // delete this->client;
+}
+
+int S3Client::get_object(const std::string &bucket, const std::string &key, int &error_type, std::string &error_message, std::string &result, std::string &range)
+{
+    Aws::S3::Model::GetObjectRequest object_request;
+    object_request.SetBucket(bucket.c_str());
+    object_request.SetKey(key.c_str());
+    if ("" != range)
+        object_request.SetRange(std::string("bytes="+range).c_str());
+
+    /* 这里真正请求数据 */
+    auto get_object_outcome = this->client->GetObject(object_request);
+
+    if (get_object_outcome.IsSuccess())
+    {
+        auto &&get_result = get_object_outcome.GetResultWithOwnership();
+        auto content_length = get_result.GetContentLength();
+        auto &retrieved_object = get_result.GetBody();
+
+        result.resize(content_length);
+        long read_offset = 0;
+        while (true)
+        {
+            retrieved_object.read(&result[read_offset], content_length - read_offset);
+            auto read_count = retrieved_object.gcount();
+            read_offset += read_count;
+            if (read_offset >= content_length || read_count == 0)
+            {
+                break;
+            }
+        }
+        if (read_offset != content_length)
+        {
+            //todo
+        }
+
+        return 0;
+    }
+    else
+    {
+        auto error = get_object_outcome.GetError();
+        auto message = error.GetMessage();
+        if (!message.empty())
+        {
+            error_message = message.c_str();
+        }
+        error_type = static_cast<int>(error.GetErrorType());
+        return -1;
+    }
+    return 0;
+}
+
+int S3Client::put_object(const std::string &bucket, const std::string &key, const std::string &data, int &error_type, std::string &error_message)
+{
+    const std::shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::StringStream>("");
+    *input_data << data;
+
+    Aws::S3::Model::PutObjectRequest request;
+    request.SetBucket(bucket.c_str());
+    request.SetKey(key.c_str());
+    request.SetBody(input_data);
+    auto outcome = this->client->PutObject(request);
+    if (outcome.IsSuccess())
+    {
+        return 0;
+    }
+    else
+    {
+        auto error = outcome.GetError();
+        auto message = error.GetMessage();
+        if (!message.empty())
+        {
+            error_message = message.c_str();
+        }
+        error_type = static_cast<int>(error.GetErrorType());
+        return -1;
+    }
+}
+
+
+int S3Client::multipart_download_concurrency(const std::string &bucket, const std::string &key, const std::string &filename, int &error_type, std::string &error_message)
+{
+    auto executor = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>("executor", this->threads_num);
+    Aws::Transfer::TransferManagerConfiguration transfer_config(executor.get());
+    transfer_config.s3Client = this->client;
+    auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config);
+
+    auto downloadHandle = transfer_manager->DownloadFile(bucket.c_str(),
+        key.c_str(),
+        [=](){
+            return Aws::New<Aws::FStream>("S3_DOWNLOAD", filename.c_str(), std::ios_base::out | std::ios_base::binary);
+        });
+
+    downloadHandle->WaitUntilFinished();// Block calling thread until download is complete.
+    auto downStat = downloadHandle->GetStatus();
+    if (downStat != Aws::Transfer::TransferStatus::COMPLETED)
+    {
+        auto error = downloadHandle->GetLastError();
+        error_message = error.GetMessage().c_str();
+        error_type = static_cast<int>(error.GetErrorType());
+        return -1;
+    }
+
+    return 0;
+}
+
+int S3Client::multipart_upload_concurrency(const std::string bucket, const std::string key, const std::string filename, int &error_type, std::string &error_message)
+{
+    auto executor = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>("executor", this->threads_num);
+    Aws::Transfer::TransferManagerConfiguration transfer_config(executor.get());
+    transfer_config.s3Client = this->client;
+    auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config);
+
+    auto uploadHandle = transfer_manager->UploadFile(filename.c_str(), bucket.c_str(), key.c_str(), "text/plain", Aws::Map<Aws::String, Aws::String>());
+    uploadHandle->WaitUntilFinished();
+    bool success = uploadHandle->GetStatus() == Aws::Transfer::TransferStatus::COMPLETED; 
+
+    if (!success)
+    {
+        auto error = uploadHandle->GetLastError();           
+        error_message = error.GetMessage().c_str();
+        error_type = static_cast<int>(error.GetErrorType());
+        return -1;
+    }
+    else
+    {
+        return 0;
+    }
+}
+
+int S3Client::delete_obj(const std::string &bucket, const std::string &key, int error_type, std::string &error_message)
+{
+    Aws::S3::Model::DeleteObjectRequest request;
+    request.WithBucket(bucket.c_str()).WithKey(key.c_str());
+    auto outcome = this->client->DeleteObject(request);
+
+    return outcome.IsSuccess();
+}
+
+int S3Client::contains(const std::string &bucket, const std::string &key, int error_type, std::string &error_message)
+{
+    Aws::S3::Model::HeadObjectRequest request;
+    request.WithBucket(bucket.c_str()).WithKey(key.c_str());
+    const auto response = client->HeadObject(request);
+    auto outcome = this->client->HeadObject(request);
+
+    return outcome.IsSuccess();
+}
+
+std::vector<std::string> S3Client::list(const std::string &bucket, const std::string &key, int error_type, std::string &error_message)
+{
+    Aws::S3::Model::ListObjectsRequest request;
+    request.WithBucket(bucket.c_str()).WithPrefix(key.c_str());
+    std::vector<std::string> res;
+    auto outcome = this->client->ListObjects(request);
+
+    if (!outcome.IsSuccess()) {
+        auto error = outcome.GetError();
+        auto message = error.GetMessage();
+        if (!message.empty())
+        {
+            error_message = message.c_str();
+        }
+        error_type = static_cast<int>(error.GetErrorType());
+        return res;
+    }
+    else {
+        Aws::Vector<Aws::S3::Model::Object> objects =
+                outcome.GetResult().GetContents();
+
+        for (Aws::S3::Model::Object &object: objects) {
+            std::string full_path = object.GetKey().c_str();
+            int pos = full_path.find('/', key.size());
+            int len = pos - key.size();
+            if (-1 != pos) {
+                len += 1;
+            }
+            std::string first_path = full_path.substr(key.size(), len);
+            res.push_back(first_path);
+        }
+    }
+
+    return res;
+}

petrel-sdk/petrel-oss-cpp-sdk/s3client.h

@@ -0,0 +1,38 @@
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/ListObjectsRequest.h>
+#include <aws/s3/model/Object.h>
+#include <aws/s3/S3Client.h>
+#include <aws/core/Aws.h>
+#include <chrono>
+#include <list>
+#include <vector>
+#include <string>
+#include <fstream>
+#include <aws/s3/model/CompletedPart.h>
+#include <aws/transfer/TransferManager.h>
+
+class S3Client
+{
+public:
+    S3Client(const std::string &ak, const std::string &sk, const std::string &endpoint, bool verify_ssl, bool enable_https, bool use_dual_stack, int threads_num);
+    ~S3Client();
+
+    int get_object(const std::string &bucket, const std::string &key, int &error_type, std::string &error_message, std::string &result, std::string &range);
+    int multipart_download_concurrency(const std::string &bucket, const std::string &key, const std::string &filename, int &error_type, std::string &error_message);
+    int put_object(const std::string &bucket, const std::string &key, const std::string &data, int &error_type, std::string &error_message);
+    int multipart_upload_concurrency(const std::string bucket, const std::string key, const std::string filename, int &error_type, std::string &error_message);
+    int delete_obj(const std::string &bucket, const std::string &key, int error_type, std::string &error_message);
+    int contains(const std::string &bucket, const std::string &key, int error_type, std::string &error_message);
+    std::vector<std::string> list(const std::string &bucket, const std::string &key, int error_type, std::string &error_message);
+
+private:
+    std::shared_ptr<Aws::S3::S3Client> client;
+    int threads_num;
+};
+
+void init_api(const std::string &level);
+
+void shutdown_api();
+
+std::list<std::pair<int, std::string>> get_error_list();

petrel-sdk/petrel-oss-cpp-sdk/s3client.pxd

@@ -0,0 +1,33 @@
+from libcpp.string cimport string
+from libcpp.list cimport list
+from libcpp.pair cimport pair
+from libcpp.vector cimport vector
+
+
+cdef extern from "s3client.h":
+    cdef cppclass S3Client:
+        S3Client(
+            const string &ak,
+            const string &sk,
+            const string &endpoint,
+            bint verify_ssl,
+            bint enable_https,
+            bint use_dual_stack,
+            int threads_num,
+        ) except +
+
+        int get_object(const string &bucket, const string &key, int &error_type, string &error_message, string &result, string &range) nogil
+        int multipart_download_concurrency(const string &bucket, const string &key, const string &filename, int &error_type, string &error_message) nogil
+        int put_object(const string &bucket, const string &key, const string &data, int &error_type, string &error_message) nogil
+        int multipart_upload_concurrency(const string &bucket, const string &key, const string &filename, int &error_type, string &error_message) nogil
+        int delete_obj(const string &bucket, const string &key, int error_type, const string &error_message) nogil
+        int contains(const string &bucket, const string &key, int error_type, const string &error_message) nogil
+        vector[string] list(const string &bucket, const string &key, int error_type, const string &error_message) nogil
+
+
+
+    cdef void init_api(const string &bucket)
+
+    cdef void shutdown_api()
+    
+    cdef list[pair[int, string]] get_error_list()

petrel-sdk/petrel-oss-cpp-sdk/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup
+from Cython.Build import cythonize
+from distutils.extension import Extension
+from Cython.Distutils import build_ext
+from glob import glob
+from os.path import basename
+
+include_dirs = glob('aws-sdk-cpp/aws-cpp-sdk-*/include')
+libraries = [':{0}'.format(basename(i)) for i in glob('libs/libaws-*.so*')]
+
+setup(
+    cmdclass={'build_ext': build_ext},
+    include_dirs=include_dirs,
+    ext_modules=[
+        Extension(
+            'pys3client',
+            ['pys3client.pyx', 's3client.cpp'],
+            language="c++",
+            libraries=libraries,
+            # extra_compile_args=["-O3"],
+            extra_link_args=["-Wl,-rpath,./libs", "-L./libs"]
+        )
+    ]
+)

petrel-sdk/petrel-oss-cpp-sdk/setup.py.bak

@@ -0,0 +1,28 @@
+from setuptools import setup
+from Cython.Build import cythonize
+from distutils.extension import Extension
+from Cython.Distutils import build_ext
+from glob import glob
+from os.path import basename
+
+include_dirs = glob('aws-sdk-cpp/aws-cpp-sdk-*/include')
+# print(include_dirs)
+# include_dirs = ["~/anaconda3/include"]
+# include_dirs=["/home/PJLAB/huipeng/anaconda3/include"]
+libraries = [':{0}'.format(basename(i)) for i in glob('libs/libaws-*.so*')]
+# print(libraries)
+
+setup(
+    cmdclass={'build_ext': build_ext},
+    include_dirs=include_dirs,
+    ext_modules=[
+        Extension(
+            'pys3client',
+            ['pys3client.pyx', 's3client.cpp'],
+            language="c++",
+            libraries=libraries,
+            # extra_compile_args=["-O3"],
+            extra_link_args=["-Wl,-rpath,./libs", "-L./libs"]
+        )
+    ]
+)

petrel-sdk/petrel-oss-python-sdk/.gitignore

@@ -0,0 +1,8 @@
+petrel_client/version.py
+.vscode
+**/__pycache__
+**/*.pyc
+**/*.egg-info
+venv
+dist
+build

petrel-sdk/petrel-oss-python-sdk/.gitlab-ci.yml

@@ -0,0 +1,24 @@
+image: registry.sensetime.com/library/python:3.6
+
+
+before_script:
+  - python -V
+  - python -m venv venv
+  - source ./venv/bin/activate
+  - python setup.py sdist
+  - pip install dist/*
+
+stages:
+  - flake8
+  - unit_test
+
+flake8:
+  stage: flake8
+  script:
+    - pip install flake8
+    - flake8 --ignore E501 --exclude '.git,.tox,*.egg-info,venv,scripts,tests,examples' .  # todo remove scripts,tests,examples
+
+test:
+  stage: unit_test
+  script:
+    - python tests/run_test.py

petrel-sdk/petrel-oss-python-sdk/INSTALL

@@ -0,0 +1,2 @@
+python setup.py install
+