Project: CubeFS (Runtime / Cloud Native Storage)
File: blobnode.md

`BlobNode` is a single-machine storage engine module, mainly responsible for organizing data to disk, reading data from disk, deleting data from disk, and executing background tasks. BlobNode configuration is based on the , and the following configuration instructions mainly apply to private configuration for BlobNode. | Configuration Item | Description | Required | |:|:|:| | Public Configuration | Such as server ports, running logs, and audit logs, refer to the section | Yes | | disks | List of disk paths to register | Yes | | disable_sync | Whether to disable disk sync. A value of true means that sync is disabled, which can improve write performance. | No | | rack | Rack number. This field is required when clustermgr opens rack isolation. | No | | host | Blobnode service address for this machine, which needs to be registered with clustermgr | Yes | | mustmountpoint | Verify whether the registered path is a mount point. It is recommended to enable this in production environments. | No | | data_qos | Data QoS hierarchical flow control. It is recommended to enable this in production environments. | No | | meta_config | Metadata-related configuration, including the cache size of RocksDB. | No | | clustermgr | Clustermgr service address information, etc. | Yes | ```json { "cluster_id": "cluster ID", "idc": "IDC", "rack": "rack", "host": "service address IP:port", "droppedbidrecord": { "dir": "directory for dropped bid records", "chunkbits": "n: file size, 2^n bytes" }, "disks": [ { "auto_format": "whether to automatically create directories", "disable_sync": "whether to disable disk sync", "path": "data storage directory", "max_chunks": "maximum number of chunks per disk" }, { "auto_format": "same as above", "disable_sync": "same as above", "path": "same as above: each disk needs to be configured and registered with clustermgr", "max_chunks": "same as above" } ], "disk_config": { "diskreservedspace_B": "reserved space per disk. The available space will be reduced by this value. Default is 60GB.", "compactreservedspace_B": "reserved space for compression per disk. Default is 20GB.", "chunkprotectionM": "protection period for released chunks. If the last modification time is within this period, release is not allowed.", "chunkcompactinterval_S": "interval for the compression task", "chunkcleaninterval_S": "interval for the chunk cleaning task", "chunkgccreatetimeprotection_M": "protection period for the creation time of chunks during cleaning", "chunkgcmodifytimeprotection_M": "protection period for the modification time of chunks during cleaning", "diskusageinterval_S": "interval for updating disk space usage", "diskcleantrashintervalS": "interval for cleaning disk garbage data", "disktrashprotection_M": "protection period for disk garbage data", "allowcleantrash": "whether to allow cleaning garbage", "disablemodifyin_compacting": "whether to disallow chunk modification during compression", "compactminsize_threshold": "minimum chunk size for compression", "compacttriggerthreshold": "chunk size at which compression is triggered", "compactemptyrate_threshold": "hole rate at which chunks can be compressed", "needcompactcheck": "whether to check the consistency of data before and after compression", "allowforcecompact": "whether to allow forced compression through the interface, bypassing compression conditions", "compactbatchsize": "number of bids per batch for compression", "mustmountpoint": "whether the data storage directory must be a mount point", "metricreportinterval_S": "interval for metric reporting", "data_qos": { "diskbandwidthMBPS": "bandwidth threshold per disk. When the bandwidth reaches this value, the bandwidth limit for each level is adjusted to bandwidth_MBPS*factor.", "disk_iops": "IOPS threshold per disk. When the IOPS reaches this value, the IOPS limit for each level is adjusted to the level configuration's

"flow_conf": { "level0": { "bandwidth_MBPS": "(level0 is for user read/write IO) bandwidth limit in MB/s", "iops": "IOPS limit", "factor": "limiting factor" }, "level1": { "bandwidth_MBPS": "(level1 is for shard repair IO) bandwidth limit in MB/s", "iops": "same as above", "factor": "same as above" }, "level2": { "bandwidth_MBPS": "(level2 is for disk repair, delete, and compact IO) bandwidth limit in MB/s", "iops": "same as above", "factor": "same as above" }, "level3": { "bandwidth_MBPS": "(level3 is for balance, drop, and manual migrate IO) bandwidth limit in MB/s", "iops": "same as above", "factor": "same as above" }, "level4": { "bandwidth_MBPS": "(level4 is for inspect IO) bandwidth limit in MB/s", "iops": "same as above", "factor": "same as above" } } } }, "meta_config": { "metarootprefix": "unified meta data storage directory configuration. Can be configured to an SSD disk to improve metadata read/write speed. Not configured by default.", "batchprocesscount": "number of batch processing requests for metadata, including deletion and writing", "support_inline": "whether to enable inline writing of small files to metadata storage RocksDB", "tinyfilethresholdB": "threshold for small files, can be set to less than or equal to 128k", "writepriratio": "ratio of write requests for metadata batch processing. The specific number is batchprocesscount*writepriratio", "sync": "whether to enable disk sync", "rocksdb_option": { "lrucache": "cache size", "writebuffersize": "RocksDB write buffer size" }, "meta_qos": { "level0": { "iops": "IOPS limit for user read/write IO, metadata flow control configuration" }, "level1": { "iops": "IOPS limit for shard repair IO" }, "level2": { "iops": "IOPS limit for disk repair, delete, and compact IO" }, "level3": { "iops": "IOPS limit for balance, drop, and manual migrate IO" }, "level4": { "iops": "IOPS limit for inspect IO" } } }, "clustermgr": { "hosts": "clustermgr service address" }, "blobnode": { "clienttimeoutms": "timeout for blobnode client used in background tasks" }, "scheduler": { "hostsyncinterval_ms": "backend node synchronization time for scheduler client used in background tasks" }, "chunkprotectionperiod_S": "protection period for expired epoch chunks based on creation time", "putqpslimitperdisk": "concurrency control for single-disk writes", "getqpslimitperdisk": "concurrency control for single-disk reads", "getqpslimitperkey": "concurrency control for reads of a single shard", "deleteqpslimitperdisk": "concurrency control for single-disk deletions", "shardrepairconcurrency": "concurrency control for background task shard repair", "flock_filename": "process file lock path" } ``` ```json { "bind_addr": ":8889", "log": { "level": 2 }, "cluster_id": 10001, "idc": "bjht", "rack": "HT02-B11-F4-402-0406", "host": "http://10.39.34.185:8889", "droppedbidrecord": { "dir": "/home/service/ebs-blobnode/package/droppedbids/", "chunkbits": 29 }, "disks": [ {"autoformat": true,"disablesync": true,"path": "/home/service/var/data1"}, {"autoformat": true,"disablesync": true,"path": "/home/service/var/data2"} ], "disk_config": { "chunkcleaninterval_S": 60, "chunkprotectionM": 30, "diskcleantrashintervalS": 60, "disktrashprotection_M": 1440, "metricreportinterval_S": 300, "needcompactcheck": true, "allowforcecompact": true, "allowcleantrash": true, "mustmountpoint": true, "data_qos": { "diskbandwidthMBPS": 200, "disk_iops": 8000, "flow_conf": { "level0": { "bandwidth_MBPS": 200, "iops": 4000, "factor": 1 }, "level1": { "bandwidth_MBPS": 40, "iops": 2000, "factor": 0.5 }, "level2": { "bandwidth_MBPS": 40, "iops": 2000, "factor": 0.5 }, "level3": { "bandwidth_MBPS": 40, "iops": 2000, "factor": 0.5 }, "level4": { "bandwidth_MBPS": 20, "iops": 1000, "factor": 0.5 } } } }, "meta_config": { "sync": false, "rocksdb_option": { "lrucache": 268435456, "writebuffersize": 52428800 }, "meta_qos": { "level0": { "iops": 8000 }, "level1": { "iops": 8000 }, "level2": { "iops": 8000 }, "level3": { "iops": 8000 }, "level4": { "iops": 8000 } } }, "clustermgr": { "hosts": [ "http://10.39.30.78:9998", "http://10.39.32.224:9998", "http://10.39.32.234:9998" ], "transport_config": { "maxconnsper_host": 4, "auth": { "enable_auth": false, "secret": "b2e5e2ed-6fca-47ce-bfbc-5e8f0650603b" } } }, "blobnode": { "clienttimeoutms": 5000 }, "scheduler": { "hostsyncinterval_ms": 3600000 }, "chunkprotectionperiod_S": 600, "putqpslimitperdisk": 1024, "getqpslimitperdisk": 1024, "getqpslimitperkey": 1024, "deleteqpslimitperdisk": 64, "shardrepairconcurrency": 100, "flockfilename": "/home/service/ebs-blobnode/package/run/blobnode.0.flock", "auditlog": { "logdir": "/home/service/ebs-blobnode/_package/run/auditlog/ebs-blobnode", "chunkbits": 29, "logfilesuffix": ".log", "backup": 10, "filters": [ {"should": {"match": {"path": ["list", "metrics", "/shard/get/"]}}} ], "metric_config": { "idc": "bjht", "service": "BLOBNODE", "team": "ocs", "enablehttpmethod": true, "enablereqlength_cnt": true, "enableresplength_cnt": true, "enablerespduration": true, "maxapilevel": 3, "size_buckets": [ 0,