Repository: dragonflyoss/Dragonfly2
Pull request #910 (2021-12-09 10:19:22)

feat: data directory
Signed-off-by: Gaius <gaius.qi@gmail.com>

<!--- Provide a general summary of your changes in the Title above -->

## Description
- Add data directory.
<!--- Describe your changes in detail -->

## Related Issue

<!--- This project only accepts pull requests related to open issues -->
<!--- If suggesting a new feature or change, please discuss it in an issue first -->
<!--- If fixing a bug, there should be an issue describing it with steps to reproduce -->
<!--- Please link to the issue here: -->

## Motivation and Context

<!--- Why is this change required? What problem does it solve? -->

## Screenshots (if appropriate):

## Types of changes

<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
- [ ] Documentation Update (if none of the other choices apply)

## Checklist:

<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->

- [x] Code compiles correctly.
- [x] My code follows the code style of this project.
- [x] My change requires a change to the documentation.
- [x] I have updated the documentation accordingly.
- [x] I have read the **CONTRIBUTING** document.
- [x] I have added tests to cover my changes.
- [x] All new and existing tests passed.



Patch:
diff --git a/client/config/dfget.go b/client/config/dfget.go
index eb47cdb2..4308b93a 100644
--- a/client/config/dfget.go
+++ b/client/config/dfget.go
@@ -96,9 +96,6 @@ type ClientOption struct {
 	// ShowProgress shows progress bar, it's conflict with `--console`.
 	ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"`
 
-	// DaemonSockPath is dfget daemon socket path.
-	DaemonSockPath string `yaml:"daemonSockPath,omitempty" mapstructure:"daemonSockPath,omitempty"`
-
 	// LogDir is log directory of dfget.
 	LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"`
 
diff --git a/client/config/peerhost.go b/client/config/peerhost.go
index e321697e..bcc976d0 100644
--- a/client/config/peerhost.go
+++ b/client/config/peerhost.go
@@ -53,6 +53,7 @@ type DaemonOption struct {
 	WorkHome    string `mapstructure:"workHome" yaml:"workHome"`
 	CacheDir    string `mapstructure:"cacheDir" yaml:"cacheDir"`
 	LogDir      string `mapstructure:"logDir" yaml:"logDir"`
+	DataDir     string `mapstructure:"dataDir" yaml:"dataDir"`
 	KeepStorage bool   `mapstructure:"keepStorage" yaml:"keepStorage"`
 
 	Scheduler    SchedulerOption `mapstructure:"scheduler" yaml:"scheduler"`
diff --git a/client/daemon/peer/piece_manager.go b/client/daemon/peer/piece_manager.go
index 4e5f41e3..59e4c836 100644
--- a/client/daemon/peer/piece_manager.go
+++ b/client/daemon/peer/piece_manager.go
@@ -414,6 +414,7 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
 						},
 						ContentLength:  contentLength,
 						GenPieceDigest: true,
+						TotalPieces:    pt.GetTotalPieces(),
 					}); err != nil {
 					log.Errorf("update task failed %s", err)
 				}
diff --git a/cmd/dfget/cmd/daemon.go b/cmd/dfget/cmd/daemon.go
index 59d4c205..fcdbd53a 100644
--- a/cmd/dfget/cmd/daemon.go
+++ b/cmd/dfget/cmd/daemon.go
@@ -110,6 +110,10 @@ func initDaemonDfpath(cfg *config.DaemonOption) (dfpath.Dfpath, error) {
 		options = append(options, dfpath.WithLogDir(cfg.LogDir))
 	}
 
+	if cfg.DataDir != "" {
+		options = append(options, dfpath.WithDataDir(cfg.DataDir))
+	}
+
 	return dfpath.New(options...)
 }
 
diff --git a/cmd/dfget/cmd/root.go b/cmd/dfget/cmd/root.go
index afffd412..8ea5895d 100644
--- a/cmd/dfget/cmd/root.go
+++ b/cmd/dfget/cmd/root.go
@@ -163,8 +163,6 @@ func init() {
 
 	flagSet.String("log-dir", dfgetConfig.LogDir, "Dfget log directory")
 
-	flagSet.String("daemon-sock-path", dfgetConfig.DaemonSockPath, "The socket path of dfget daemon")
-
 	// Bind cmd flags
 	if err := viper.BindPFlags(flagSet); err != nil {
 		panic(errors.Wrap(err, "bind dfget flags to viper"))
diff --git a/docs/en/deployment/configuration/dfget.yaml b/docs/en/deployment/configuration/dfget.yaml
index fe76b354..fb1c1b4b 100644
--- a/docs/en/deployment/configuration/dfget.yaml
+++ b/docs/en/deployment/configuration/dfget.yaml
@@ -20,6 +20,11 @@ cacheDir: ""
 # in macos(just for testing), default value is /Users/$USER/.dragonfly/logs
 logDir: ""
 
+# dataDir is the download data storage directory
+# in linux, default value is /var/bin/dragonfly
+# in macos(just for testing), default value is /Users/$USER/.dragonfly/data
+dataDir: ""
+
 # when daemon exit, keep peer task data or not
 # it is usefully when upgrade daemon service, all local cache will be saved
 # default is false
diff --git a/docs/zh-CN/deployment/configuration/cdn.yaml b/docs/zh-CN/deployment/configuration/cdn.yaml
index 91827d89..e9e2d544 100644
--- a/docs/zh-CN/deployment/configuration/cdn.yaml
+++ b/docs/zh-CN/deployment/configuration/cdn.yaml
@@ -47,6 +47,11 @@ base:
   # 默认值：disk
   storageMode: disk
 
+  # cdn 日志目录
+  # linux 上默认目录 /var/log/dragonfly
+  # macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/logs
+  logDir: ""
+
   # CDN 连接的 manager，可以不指定。
   # 各项配置默认值如下。如果 addr 为空字符串，CDN将不会连接manager。
   manager:
diff --git a/docs/zh-CN/deployment/configuration/dfget.yaml b/docs/zh-CN/deployment/configuration/dfget.yaml
index 6efe9dd0..b56273fc 100644
--- a/docs/zh-CN/deployment/configuration/dfget.yaml
+++ b/docs/zh-CN/deployment/configuration/dfget.yaml
@@ -5,15 +5,25 @@ aliveTime: 0s
 # daemon gc 间隔
 gcInterval: 1m0s
 
-# daemon 数据目录
-# linux 上默认目录为 $HOME/.dragonfly/dfget-daemon/
-# macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/dfdaemon/
-dataDir: /var/lib/dragonfly
-
 # daemon 工作目录
-# linux 上默认目录 $HOME/.dragonfly/dfget-daemon/
-# macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/dfdaemon/
-workHome: /var/lib/dragonfly
+# linux 上默认目录 /usr/local/dragonfly
+# macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly
+workHome: ""
+
+# daemon 动态配置缓存目录
+# linux 上默认目录 /var/cache/dragonfly
+# macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/cache
+cacheDir: ""
+
+# daemon 日志目录
+# linux 上默认目录 /var/log/dragonfly
+# macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/logs
+logDir: ""
+
+# daemon 数据目录
+# linux 上默认目录为 /var/lib/dragonfly
+# macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/data/
+dataDir: ""
 
 # 当 daemon 退出是, 是否保存缓存数据
 # 保留缓存数据在升级 daemon 的时候比较有用
diff --git a/docs/zh-CN/deployment/configuration/scheduler.yaml b/docs/zh-CN/deployment/configuration/scheduler.yaml
index 1d6108ad..6ffd7821 100644
--- a/docs/zh-CN/deployment/configuration/scheduler.yaml
+++ b/docs/zh-CN/deployment/configuration/scheduler.yaml
@@ -55,6 +55,15 @@ server:
   # ListenPort is the ip and port scheduler server listens on.
   # default: 8002
   port:
+  # daemon 动态配置缓存目录
+  # linux 上默认目录 /var/cache/dragonfly
+  # macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/cache
+  cacheDir: ""
+
+  # daemon 日志目录
+  # linux 上默认目录 /var/log/dragonfly
+  # macos(仅开发、测试), 默认目录是 /Users/$USER/.dragonfly/logs
+  logDir: ""
 
 # 动态数据配置
 dynConfig:
@@ -123,7 +132,6 @@ jaeger: ""
 # tracer 中使用的 service-name
 # 默认值：dragonfly-cdn
 service-name: dragonfly-scheduler
-
 # 开启数据收集服务
 # metrics:
 #  # 数据服务地址
diff --git a/internal/dfpath/dfpath.go b/internal/dfpath/dfpath.go
index 46209068..75921fb8 100644
--- a/internal/dfpath/dfpath.go
+++ b/internal/dfpath/dfpath.go
@@ -78,6 +78,13 @@ func WithLogDir(dir string) Option {
 	}
 }
 
+// WithDataDir set download data directory
+func WithDataDir(dir string) Option {
+	return func(d *dfpath) {
+		d.dataDir = dir
+	}
+}
+
 // New returns a new dfpath interface
 func New(options ...Option) (Dfpath, error) {
 	cache.Do(func() {
@@ -85,13 +92,13 @@ func New(options ...Option) (Dfpath, error) {
 			workHome: DefaultWorkHome,
 			cacheDir: DefaultCacheDir,
 			logDir:   DefaultLogDir,
+			dataDir:  DefaultDataDir,
 		}
 
 		for _, opt := range options {
 			opt(d)
 		}
 
-		d.dataDir = filepath.Join(d.workHome, "data")
 		d.pluginDir = filepath.Join(d.workHome, "plugins")
 		d.daemonSockPath = filepath.Join(d.workHome, "daemon.sock")
 		d.daemonLockPath = filepath.Join(d.workHome, "daemon.lock")
diff --git a/internal/dfpath/dfpath_darwin.go b/internal/dfpath/dfpath_darwin.go
index 3905daee..41590a72 100644
--- a/internal/dfpath/dfpath_darwin.go
+++ b/internal/dfpath/dfpath_darwin.go
@@ -29,3 +29,4 @@ var DefaultWorkHome = filepath.Join(basic.HomeDir, ".dragonfly")
 var DefaultCacheDir = filepath.Join(DefaultWorkHome, "cache")
 var DefaultConfigDir = filepath.Join(DefaultWorkHome, "config")
 var DefaultLogDir = filepath.Join(DefaultWorkHome, "logs")
+var DefaultDataDir = filepath.Join(DefaultWorkHome, "data")
diff --git a/internal/dfpath/dfpath_linux.go b/internal/dfpath/dfpath_linux.go
index 5db3041f..589f72fc 100644
--- a/internal/dfpath/dfpath_linux.go
+++ b/internal/dfpath/dfpath_linux.go
@@ -23,3 +23,4 @@ var DefaultWorkHome = "/usr/local/dragonfly"
 var DefaultCacheDir = "/var/cache/dragonfly"
 var DefaultConfigDir = "/etc/dragonfly"
 var DefaultLogDir = "/var/log/dragonfly"
+var DefaultDataDir = "/var/lib/dragonfly"


Test patch:
diff --git a/client/config/peerhost_test.go b/client/config/peerhost_test.go
index fff4b46e..6be66729 100644
--- a/client/config/peerhost_test.go
+++ b/client/config/peerhost_test.go
@@ -225,6 +225,9 @@ func TestPeerHostOption_Load(t *testing.T) {
 			Duration: 60000000000,
 		},
 		WorkHome:    "/tmp/dragonfly/dfdaemon/",
+		DataDir:     "/var/lib/dragonfly/",
+		LogDir:      "/var/log/dragonfly/",
+		CacheDir:    "/var/cache/dragonfly/",
 		KeepStorage: false,
 		Scheduler: SchedulerOption{
 			Manager: ManagerOption{
diff --git a/client/config/testdata/config/daemon.yaml b/client/config/testdata/config/daemon.yaml
index 63d24d35..07a02f2a 100644
--- a/client/config/testdata/config/daemon.yaml
+++ b/client/config/testdata/config/daemon.yaml
@@ -1,6 +1,9 @@
 aliveTime: 0s
 gcInterval: 1m0s
 workHome: /tmp/dragonfly/dfdaemon/
+dataDir: /var/lib/dragonfly/
+logDir: /var/log/dragonfly/
+cacheDir: /var/cache/dragonfly/
 keepStorage: false
 scheduler:
   manager:
diff --git a/client/daemon/peer/piece_manager_test.go b/client/daemon/peer/piece_manager_test.go
index df7a2e49..2c6f059c 100644
--- a/client/daemon/peer/piece_manager_test.go
+++ b/client/daemon/peer/piece_manager_test.go
@@ -33,6 +33,7 @@ import (
 	"github.com/golang/mock/gomock"
 	testifyassert "github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
 
 	"d7y.io/dragonfly/v2/client/clientutil"
 	"d7y.io/dragonfly/v2/client/config"
@@ -83,7 +84,7 @@ func TestPieceManager_DownloadSource(t *testing.T) {
 		checkDigest       bool
 	}{
 		{
-			name:              "multiple pieces with content length",
+			name:              "multiple pieces with content length, check digest",
 			pieceSize:         1024,
 			checkDigest:       true,
 			withContentLength: true,
@@ -95,7 +96,7 @@ func TestPieceManager_DownloadSource(t *testing.T) {
 			withContentLength: true,
 		},
 		{
-			name:              "multiple pieces without content length",
+			name:              "multiple pieces without content length, check digest",
 			pieceSize:         1024,
 			checkDigest:       true,
 			withContentLength: false,
@@ -131,12 +132,18 @@ func TestPieceManager_DownloadSource(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			/********** prepare test start **********/
 			mockPeerTask := NewMockTask(ctrl)
+			var totalPieces = &atomic.Int32{}
 			mockPeerTask.EXPECT().SetContentLength(gomock.Any()).AnyTimes().DoAndReturn(
 				func(arg0 int64) error {
 					return nil
 				})
 			mockPeerTask.EXPECT().SetTotalPieces(gomock.Any()).AnyTimes().DoAndReturn(
 				func(arg0 int32) {
+					totalPieces.Store(arg0)
+				})
+			mockPeerTask.EXPECT().GetTotalPieces().AnyTimes().DoAndReturn(
+				func() int32 {
+					return totalPieces.Load()
 				})
 			mockPeerTask.EXPECT().GetPeerID().AnyTimes().DoAndReturn(
 				func() string {
diff --git a/internal/dfpath/dfpath_test.go b/internal/dfpath/dfpath_test.go
index aeb048e2..ca112b3a 100644
--- a/internal/dfpath/dfpath_test.go
+++ b/internal/dfpath/dfpath_test.go
@@ -37,6 +37,7 @@ func TestNew(t *testing.T) {
 				assert.Equal(d.WorkHome(), DefaultWorkHome)
 				assert.Equal(d.CacheDir(), DefaultCacheDir)
 				assert.Equal(d.LogDir(), DefaultLogDir)
+				assert.Equal(d.DataDir(), DefaultDataDir)
 			},
 		},
 		{
@@ -49,6 +50,7 @@ func TestNew(t *testing.T) {
 				assert.Equal(d.WorkHome(), DefaultWorkHome)
 				assert.Equal(d.CacheDir(), DefaultCacheDir)
 				assert.Equal(d.LogDir(), DefaultLogDir)
+				assert.Equal(d.DataDir(), DefaultDataDir)
 			},
 		},
 		{
@@ -61,6 +63,7 @@ func TestNew(t *testing.T) {
 				assert.Equal(d.WorkHome(), DefaultWorkHome)
 				assert.Equal(d.CacheDir(), DefaultCacheDir)
 				assert.Equal(d.LogDir(), DefaultLogDir)
+				assert.Equal(d.DataDir(), DefaultDataDir)
 			},
 		},
 		{
@@ -73,6 +76,7 @@ func TestNew(t *testing.T) {
 				assert.Equal(d.WorkHome(), DefaultWorkHome)
 				assert.Equal(d.CacheDir(), DefaultCacheDir)
 				assert.Equal(d.LogDir(), DefaultLogDir)
+				assert.Equal(d.DataDir(), DefaultDataDir)
 			},
 		},
 	}