Repository: dragonflyoss/Dragonfly2
Pull request #1466 (2022-07-15 07:59:26)

fix:upload_manager write header immediately when it is ready
Signed-off-by: bigerous <bigerous@qq.com>

<!--- Provide a general summary of your changes in the Title above -->

uploader should write header immediately, prevent client disconnecting after limiter.Wait() due to response header timeout

<!--- Describe your changes in detail -->

## Related Issue

<!--- This project only accepts pull requests related to open issues -->
<!--- If suggesting a new feature or change, please discuss it in an issue first -->
<!--- If fixing a bug, there should be an issue describing it with steps to reproduce -->
<!--- Please link to the issue here: -->

## Motivation and Context

<!--- Why is this change required? What problem does it solve? -->

## Screenshots (if appropriate)

## Types of changes

<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
- [ ] Documentation Update (if none of the other choices apply)

## Checklist

<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->

- [ ] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.
- [ ] I have read the **CONTRIBUTING** document.
- [ ] I have added tests to cover my changes.



Patch:
diff --git a/client/config/dfget.go b/client/config/dfget.go
index db810d39..88b4cf26 100644
--- a/client/config/dfget.go
+++ b/client/config/dfget.go
@@ -234,16 +234,9 @@ func (cfg *ClientOption) checkOutput() error {
 	}
 
 	f, err := os.Stat(cfg.Output)
-	// when recursive download, need a directory
-	if cfg.Recursive {
-		if err == nil && !f.IsDir() {
-			return fmt.Errorf("path[%s] is file but requires directory path", cfg.Output)
-		}
-	} else {
-		// when not recursive download, need a file
-		if err == nil && f.IsDir() {
-			return fmt.Errorf("path[%s] is directory but requires file path", cfg.Output)
-		}
+	// when not recursive download, need a file
+	if !cfg.Recursive && err == nil && f.IsDir() {
+		return fmt.Errorf("path[%s] is directory but requires file path", cfg.Output)
 	}
 
 	// check permission
diff --git a/client/daemon/storage/local_storage.go b/client/daemon/storage/local_storage.go
index ffbe9ace..31b98e51 100644
--- a/client/daemon/storage/local_storage.go
+++ b/client/daemon/storage/local_storage.go
@@ -101,6 +101,7 @@ func (t *localTaskStore) WritePiece(ctx context.Context, req *WritePieceRequest)
 	t.RLock()
 	if piece, ok := t.Pieces[req.Num]; ok {
 		t.RUnlock()
+		t.Debugf("piece %d already exist,ignore writing piece", req.Num)
 		// discard already downloaded data for back source
 		n, err := io.CopyN(io.Discard, req.Reader, piece.Range.Length)
 		if err != nil && err != io.EOF {
@@ -109,6 +110,11 @@ func (t *localTaskStore) WritePiece(ctx context.Context, req *WritePieceRequest)
 		if n != piece.Range.Length {
 			return n, ErrShortRead
 		}
+		// GenMetadata need to be called when using concurrent download, a Counter will increase in GenMetadata
+		if req.GenMetadata != nil {
+			req.GenMetadata(n)
+		}
+
 		return piece.Range.Length, nil
 	}
 	t.RUnlock()
diff --git a/client/daemon/storage/local_storage_subtask.go b/client/daemon/storage/local_storage_subtask.go
index 25e4af37..f552a68e 100644
--- a/client/daemon/storage/local_storage_subtask.go
+++ b/client/daemon/storage/local_storage_subtask.go
@@ -48,6 +48,7 @@ func (t *localSubTaskStore) WritePiece(ctx context.Context, req *WritePieceReque
 	t.RLock()
 	if piece, ok := t.Pieces[req.Num]; ok {
 		t.RUnlock()
+		t.Debugf("piece %d already exist,ignore writing piece", req.Num)
 		// discard already downloaded data for back source
 		n, err := io.CopyN(io.Discard, req.Reader, piece.Range.Length)
 		if err != nil && err != io.EOF {
@@ -56,6 +57,11 @@ func (t *localSubTaskStore) WritePiece(ctx context.Context, req *WritePieceReque
 		if n != piece.Range.Length {
 			return n, ErrShortRead
 		}
+		// GenMetadata need to be called when using concurrent download, a Counter will increase in GenMetadata
+		if req.GenMetadata != nil {
+			req.GenMetadata(n)
+		}
+
 		return piece.Range.Length, nil
 	}
 	t.RUnlock()
diff --git a/client/daemon/upload/upload_manager.go b/client/daemon/upload/upload_manager.go
index 5a7da1be..c92edb11 100644
--- a/client/daemon/upload/upload_manager.go
+++ b/client/daemon/upload/upload_manager.go
@@ -212,6 +212,9 @@ func (um *uploadManager) getDownload(ctx *gin.Context) {
 	}
 	defer closer.Close()
 
+	// write header immediately, prevent client disconnecting after limiter.Wait() due to response header timeout
+	ctx.Writer.WriteHeaderNow()
+
 	if um.Limiter != nil {
 		if err = um.Limiter.WaitN(ctx, int(rg[0].Length)); err != nil {
 			log.Errorf("get limit failed: %s", err)
diff --git a/client/dfget/dfget.go b/client/dfget/dfget.go
index f6c060af..ca34b1bb 100644
--- a/client/dfget/dfget.go
+++ b/client/dfget/dfget.go
@@ -29,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gammazero/deque"
 	"github.com/go-http-utils/headers"
 	"github.com/schollz/progressbar/v3"
 
@@ -266,11 +267,7 @@ func newProgressBar(max int64) *progressbar.ProgressBar {
 		}))
 }
 
-func accept(u string, parent, sub string, level uint, accept, reject string) bool {
-	if !checkDirectoryLevel(parent, sub, level) {
-		logger.Debugf("url %s not accept by level: %d", u, level)
-		return false
-	}
+func accept(u string, accept, reject string) bool {
 	if !acceptRegex(u, accept) {
 		logger.Debugf("url %s not accept by regex: %s", u, accept)
 		return false
@@ -282,14 +279,6 @@ func accept(u string, parent, sub string, level uint, accept, reject string) boo
 	return true
 }
 
-func checkDirectoryLevel(parent, sub string, level uint) bool {
-	if level == 0 {
-		return true
-	}
-	dirs := strings.Split(strings.Trim(strings.TrimLeft(sub, parent), "/"), "/")
-	return uint(len(dirs)) <= level
-}
-
 func acceptRegex(u string, accept string) bool {
 	if accept == "" {
 		return true
@@ -304,44 +293,64 @@ func rejectRegex(u string, reject string) bool {
 	return regexp.MustCompile(reject).Match([]byte(u))
 }
 
+// recursiveDownload breadth-first download all resources
 func recursiveDownload(ctx context.Context, client daemonclient.DaemonClient, cfg *config.DfgetConfig) error {
-	request, err := source.NewRequestWithContext(ctx, cfg.URL, parseHeader(cfg.Header))
-	if err != nil {
-		return err
-	}
-	dirURL, err := url.Parse(cfg.URL)
-	if err != nil {
-		return err
-	}
-	logger.Debugf("dirURL: %s", cfg.URL)
-
-	urls, err := source.List(request)
-	if err != nil {
-		return err
-	}
-	for _, u := range urls {
-		// reuse dfget config
-		c := *cfg
-		// update some attributes
-		c.Recursive, c.URL, c.Output = false, u.String(), path.Join(cfg.Output, strings.TrimPrefix(u.Path, dirURL.Path))
-		if !accept(c.URL, dirURL.Path, u.Path, cfg.RecursiveLevel, cfg.RecursiveAcceptRegex, cfg.RecursiveRejectRegex) {
-			logger.Debugf("url %s is not accepted, skip", c.URL)
-			continue
+	var queue deque.Deque[*config.DfgetConfig]
+	queue.PushBack(cfg)
+	downloadMap := map[url.URL]struct{}{}
+	for {
+		if queue.Len() == 0 {
+			break
 		}
-		if cfg.RecursiveList {
-			fmt.Printf("%s\n", u.String())
+		parentCfg := queue.PopFront()
+		if parentCfg.RecursiveLevel == 0 {
 			continue
 		}
-		// validate new dfget config
-		if err = c.Validate(); err != nil {
-			logger.Errorf("validate failed: %s", err)
+		parentCfg.RecursiveLevel--
+		request, err := source.NewRequestWithContext(ctx, parentCfg.URL, parseHeader(parentCfg.Header))
+		if err != nil {
 			return err
 		}
+		// prevent loop downloading
+		if _, exist := downloadMap[*request.URL]; exist {
+			continue
+		}
+		downloadMap[*request.URL] = struct{}{}
 
-		logger.Debugf("download %s to %s", c.URL, c.Output)
-		err = download(ctx, client, &c, logger.With("url", c.URL))
+		urlEntries, err := source.List(request)
 		if err != nil {
-			return err
+			logger.Errorf("url [%v] source lister error: %v", request.URL, err)
+		}
+		for _, urlEntry := range urlEntries {
+			childCfg := *parentCfg //create new cfg
+			childCfg.Output = path.Join(parentCfg.Output, urlEntry.Name)
+			fmt.Printf("%s\n", strings.TrimPrefix(childCfg.Output, cfg.Output))
+			u := urlEntry.URL
+			childCfg.URL = u.String()
+
+			if !accept(childCfg.URL, childCfg.RecursiveAcceptRegex, childCfg.RecursiveRejectRegex) {
+				logger.Debugf("url %s is not accepted, skip", childCfg.URL)
+				continue
+			}
+
+			logger.Debugf("download %s to %s", childCfg.URL, childCfg.Output)
+			if !urlEntry.IsDir {
+				if childCfg.RecursiveList {
+					continue
+				}
+				childCfg.Recursive = false
+				// validate new dfget config
+				if err = childCfg.Validate(); err != nil {
+					logger.Errorf("validate failed: %s", err)
+					return err
+				}
+				if err := singleDownload(ctx, client, &childCfg, logger.With("url", childCfg.URL)); err != nil {
+					return err
+				}
+			} else {
+				queue.PushBack(&childCfg)
+			}
+
 		}
 	}
 	return nil
diff --git a/go.mod b/go.mod
index 87e998c6..25a0e3b8 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-units v0.4.0
 	github.com/envoyproxy/protoc-gen-validate v0.6.7
+	github.com/gammazero/deque v0.2.0
 	github.com/gin-contrib/cors v1.3.1
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.8.1
@@ -65,6 +66,7 @@ require (
 	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
+	golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d
 	golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
 	golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b
@@ -191,7 +193,6 @@ require (
 	go.mongodb.org/mongo-driver v1.9.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d // indirect
 	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
 	golang.org/x/text v0.3.7 // indirect
diff --git a/go.sum b/go.sum
index 8a70d151..b4267022 100644
--- a/go.sum
+++ b/go.sum
@@ -249,6 +249,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
+github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA=
+github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/cors v1.3.1 h1:doAsuITavI4IOcd0Y19U4B+O0dNWihRyX//nn4sEmgA=
diff --git a/pkg/source/clients/ossprotocol/oss_source_client.go b/pkg/source/clients/ossprotocol/oss_source_client.go
index c6204207..4a866efe 100644
--- a/pkg/source/clients/ossprotocol/oss_source_client.go
+++ b/pkg/source/clients/ossprotocol/oss_source_client.go
@@ -20,7 +20,11 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"net/textproto"
+	"net/url"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -28,7 +32,7 @@ import (
 	"github.com/go-http-utils/headers"
 
 	"d7y.io/dragonfly/v2/pkg/source"
-	"d7y.io/dragonfly/v2/pkg/strings"
+	pkgstrings "d7y.io/dragonfly/v2/pkg/strings"
 )
 
 const OSSClient = "oss"
@@ -61,6 +65,7 @@ func adaptor(request *source.Request) *source.Request {
 		clonedRequest.Header.Set(oss.HTTPHeaderEtag, request.Header.Get(source.ETag))
 		clonedRequest.Header.Del(source.ETag)
 	}
+	clonedRequest.URL.Path = strings.TrimPrefix(clonedRequest.URL.Path, "/")
 	return clonedRequest
 }
 
@@ -155,7 +160,10 @@ func (osc *ossSourceClient) Download(request *source.Request) (*source.Response,
 	if err != nil {
 		return nil, fmt.Errorf("get oss bucket %s: %w", request.URL.Host, err)
 	}
-	objectResult, err := bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: request.URL.Path}, getOptions(request.Header))
+	objectResult, err := bucket.DoGetObject(
+		&oss.GetObjectRequest{ObjectKey: request.URL.Path},
+		getOptions(request.Header),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("get oss Object %s: %w", request.URL.Path, err)
 	}
@@ -202,17 +210,78 @@ func (osc *ossSourceClient) GetLastModified(request *source.Request) (int64, err
 	return t.UnixNano() / time.Millisecond.Nanoseconds(), nil
 }
 
+func (osc *ossSourceClient) List(request *source.Request) (urls []source.URLEntry, err error) {
+	client, err := osc.getClient(request.Header)
+	if err != nil {
+		return nil, fmt.Errorf("get oss client: %w", err)
+	}
+	bucket, err := client.Bucket(request.URL.Host)
+	if err != nil {
+		return nil, fmt.Errorf("get oss bucket %s: %w", request.URL.Host, err)
+	}
+	isDir, err := osc.isDirectory(bucket, request)
+	if err != nil {
+		return nil, err
+	}
+	// if request is a single file, just return
+	if !isDir {
+		return []source.URLEntry{buildURLEntry(false, request.URL)}, nil
+	}
+	// list all files and subdirectory
+	path := addTrailingSlash(request.URL.Path)
+	prefix := oss.Prefix(path)
+	marker := oss.Marker("")
+	delimiter := "/"
+	for {
+		lsRes, err := bucket.ListObjects(prefix, marker, oss.Delimiter(delimiter))
+		if err != nil {
+			return urls, fmt.Errorf("list oss object %s/%s: %w", request.URL.Host, path, err)
+		}
+		for _, object := range lsRes.Objects {
+			if object.Key != lsRes.Prefix {
+				url := *request.URL
+				url.Path = addLeadingSlash(object.Key)
+				urls = append(urls, buildURLEntry(false, &url))
+			}
+		}
+		for _, object := range lsRes.CommonPrefixes {
+			url := *request.URL
+			url.Path = addLeadingSlash(object)
+			urls = append(urls, buildURLEntry(true, &url))
+		}
+		if lsRes.IsTruncated {
+			prefix = oss.Prefix(lsRes.Prefix)
+			marker = oss.Marker(lsRes.NextMarker)
+		} else {
+			break
+		}
+	}
+	return urls, nil
+}
+
+func (osc *ossSourceClient) isDirectory(bucket *oss.Bucket, request *source.Request) (bool, error) {
+	uPath := addTrailingSlash(request.URL.Path)
+	lsRes, err := bucket.ListObjects(oss.Prefix(uPath), oss.Marker(""), oss.Delimiter("/"), oss.MaxKeys(1))
+	if err != nil {
+		return false, fmt.Errorf("list oss object %s/%s: %w", request.URL.Host, uPath, err)
+	}
+	if len(lsRes.Objects) > 0 && lsRes.Objects[0].Key == uPath && lsRes.Objects[0].Size == 0 {
+		return true, nil
+	}
+	return false, nil
+}
+
 func (osc *ossSourceClient) getClient(header source.Header) (*oss.Client, error) {
 	endpoint := header.Get(endpoint)
-	if strings.IsBlank(endpoint) {
+	if pkgstrings.IsBlank(endpoint) {
 		return nil, errors.New("endpoint is empty")
 	}
 	accessKeyID := header.Get(accessKeyID)
-	if strings.IsBlank(accessKeyID) {
+	if pkgstrings.IsBlank(accessKeyID) {
 		return nil, errors.New("accessKeyID is empty")
 	}
 	accessKeySecret := header.Get(accessKeySecret)
-	if strings.IsBlank(accessKeySecret) {
+	if pkgstrings.IsBlank(accessKeySecret) {
 		return nil, errors.New("accessKeySecret is empty")
 	}
 	clientKey := buildClientKey(endpoint, accessKeyID, accessKeySecret)
@@ -233,11 +302,40 @@ func buildClientKey(endpoint, accessKeyID, accessKeySecret string) string {
 
 func getOptions(header source.Header) []oss.Option {
 	opts := make([]oss.Option, 0, len(header))
-	for key, value := range header {
-		if key == endpoint || key == accessKeyID || key == accessKeySecret {
+	for key, values := range header {
+		if key == textproto.CanonicalMIMEHeaderKey(endpoint) || key == textproto.CanonicalMIMEHeaderKey(accessKeyID) || key == textproto.CanonicalMIMEHeaderKey(accessKeySecret) {
 			continue
 		}
+		// oss-http Header value must be string type, while http header value is []string type
+		// according to HTTP RFC2616 Multiple message-header fields with the same field-name MAY be present in a message
+		// if and only if the entire field-value for that header field is defined as a comma-separated list
+		value := strings.Join(values, ",")
 		opts = append(opts, oss.SetHeader(key, value))
 	}
 	return opts
 }
+
+func buildURLEntry(isDir bool, url *url.URL) source.URLEntry {
+	if isDir {
+		url.Path = addTrailingSlash(url.Path)
+		list := strings.Split(url.Path, "/")
+		return source.URLEntry{URL: url, Name: list[len(list)-2], IsDir: true}
+	}
+	_, name := filepath.Split(url.Path)
+	return source.URLEntry{URL: url, Name: name, IsDir: false}
+
+}
+
+func addLeadingSlash(s string) string {
+	if strings.HasPrefix(s, "/") {
+		return s
+	}
+	return "/" + s
+}
+
+func addTrailingSlash(s string) string {
+	if strings.HasSuffix(s, "/") {
+		return s
+	}
+	return s + "/"
+}
diff --git a/pkg/source/source_client.go b/pkg/source/source_client.go
index 5425ced9..3984235b 100644
--- a/pkg/source/source_client.go
+++ b/pkg/source/source_client.go
@@ -123,9 +123,28 @@ type ResourceMetadataGetter interface {
 	GetMetadata(request *Request) (*Metadata, error)
 }
 
+// URLEntry is an entry which read from url with specific protocol
+// It is used in recursive downloading
+type URLEntry struct {
+	//URL download url
+	URL *url.URL
+
+	// Name returns the name of the file (or subdirectory) described by the entry.
+	// Name will be used in recursive downloading as file name or subdirectory name
+	// This name is only the final element of the path (the base name), not the entire path.
+	// For example, Name would return "hello.go" not "home/gopher/hello.go".
+	Name string
+	// IsDir reports whether the entry describes a directory.
+	IsDir bool
+}
+
 // ResourceLister defines the interface to list all downloadable resources in request url
 type ResourceLister interface {
-	List(request *Request) (urls []*url.URL, err error)
+	// List only list current level resources in request url
+	// the request may represent a single file or a directory
+	// if the request represent a directory, the result should return all file entries and subdirectory entries below the request directory
+	// if the request represent a single file, the result should return a single file entry or empty slice
+	List(request *Request) (urls []URLEntry, err error)
 }
 
 type ClientManager interface {
@@ -348,11 +367,16 @@ func Download(request *Request) (*Response, error) {
 	return client.Download(request)
 }
 
-func List(request *Request) ([]*url.URL, error) {
+func List(request *Request) ([]URLEntry, error) {
 	client, ok := _defaultManager.GetClient(request.URL.Scheme)
 	if !ok {
 		return nil, fmt.Errorf("scheme %s: %w", request.URL.Scheme, ErrNoClientFound)
 	}
+	if wrap, ok := client.(*clientWrapper); ok {
+		if rc, ok := wrap.rc.(ResourceLister); ok {
+			return rc.List(wrap.adapter(request))
+		}
+	}
 	lister, ok := client.(ResourceLister)
 	if !ok {
 		return nil, fmt.Errorf("scheme %s: %w", request.URL.Scheme, ErrClientNotSupportList)


Test patch:
diff --git a/client/dfget/dfget_test.go b/client/dfget/dfget_test.go
index 7ab480c9..e993c7a5 100644
--- a/client/dfget/dfget_test.go
+++ b/client/dfget/dfget_test.go
@@ -18,7 +18,6 @@ package dfget
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -62,55 +61,3 @@ func Test_downloadFromSource(t *testing.T) {
 	err = downloadFromSource(context.Background(), cfg, nil)
 	assert.Nil(t, err)
 }
-
-func Test_checkDirectoryLevel(t *testing.T) {
-	testCases := []struct {
-		parent string
-		sub    string
-		level  uint
-		ok     bool
-	}{
-		{
-			parent: "/data/",
-			sub:    "/data/1/2/3/4/5",
-			level:  0,
-			ok:     true,
-		},
-		{
-			parent: "/data/",
-			sub:    "/data/1",
-			level:  1,
-			ok:     true,
-		},
-		{
-			parent: "/data/",
-			sub:    "/data/1/2",
-			level:  1,
-			ok:     false,
-		},
-		{
-			parent: "/data/",
-			sub:    "/data/1/2/3/4/5",
-			level:  5,
-			ok:     true,
-		},
-		{
-			parent: "/data/",
-			sub:    "/data/1/2/3/4/5",
-			level:  4,
-			ok:     false,
-		},
-		{
-			parent: "/d7y-test",
-			sub:    "/d7y-test/dir/file",
-			level:  1,
-			ok:     false,
-		},
-	}
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {
-			ok := checkDirectoryLevel(tc.parent, tc.sub, tc.level)
-			assert.Equal(t, tc.ok, ok)
-		})
-	}
-}