Repository: pingcap/tidb
File: pkg/objstore/s3like/store.go (Go)

Code before:
}

// FileExists check if file exists on s3 storage.
func (rs *Storage) FileExists(ctx context.Context, file string) (bool, error) {
	return rs.s3Cli.IsObjectExists(ctx, file)
}

// WalkDir traverse all the files in a dir.
//
// fn is the function called for each regular file visited by WalkDir.
// The first argument is the file path that can be used in `Open`
// function; the second argument is the size in byte of the file determined
// by path.
func (rs *Storage) WalkDir(ctx context.Context, opt *storeapi.WalkOption, fn func(string, int64) error) error {
	if opt == nil {
		opt = &storeapi.WalkOption{}
	}
	prefix := storeapi.NewPrefix(opt.SubDir).ObjectKey(opt.ObjPrefix)
	var maxKeys = 1000
	if opt.ListCount > 0 {
		maxKeys = int(opt.ListCount)
	}

	var (
		continuationToken *string
		startAfter        *string
		cliPrefix         = rs.bucketPrefix.PrefixStr()
	)
	if len(opt.StartAfter) > 0 {
		startAfter = &opt.StartAfter
	}
	for {
		res, err := rs.s3Cli.ListObjects(ctx, prefix, continuationToken, startAfter, maxKeys)
		if err != nil {
			return errors.Trace(err)
		}
		for _, r := range res.Objects {
			// when walk on specify directory, the result include client prefix,
			// which can not be reuse in other API(Open/Read) directly.
			// so we use TrimPrefix to filter Prefix for next Open/Read.
			trimmedKey := strings.TrimPrefix(r.Key, cliPrefix)
			// trim the prefix '/' to ensure that the path returned is consistent with the local storage
			trimmedKey = strings.TrimPrefix(trimmedKey, "/")
			itemSize := r.Size

			// filter out s3's empty directory items
			if itemSize <= 0 && strings.HasSuffix(trimmedKey, "/") {
				log.Info("skip empty directory which cannot be opened", zap.String("key", trimmedKey))
				continue
			}
			if err = fn(trimmedKey, itemSize); err != nil {

Review comment (suggestion):
```suggestion
		initialStartAfter        *string
```

Code after:
}

// FileExists check if file exists on s3 storage.
func (rs *Storage) FileExists(ctx context.Context, file string) (bool, error) {
	return rs.s3Cli.IsObjectExists(ctx, file)
}

// WalkDir traverse all the files in a dir.
//
// fn is the function called for each regular file visited by WalkDir.
// The first argument is the file path that can be used in `Open`
// function; the second argument is the size in byte of the file determined
// by path.
func (rs *Storage) WalkDir(ctx context.Context, opt *storeapi.WalkOption, fn func(string, int64) error) error {
	if opt == nil {
		opt = &storeapi.WalkOption{}
	}
	prefix := storeapi.NewPrefix(opt.SubDir).ObjectKey(opt.ObjPrefix)
	var maxKeys = 1000
	if opt.ListCount > 0 {
		maxKeys = int(opt.ListCount)
	}
	initialStartAfter := opt.StartAfter

	var (
		continuationToken *string
		cliPrefix         = rs.bucketPrefix.PrefixStr()
	)
	for {
		res, err := rs.s3Cli.ListObjects(ctx, prefix, initialStartAfter, continuationToken, maxKeys)
		if err != nil {
			return errors.Trace(err)
		}
		for _, r := range res.Objects {
			// when walk on specify directory, the result include client prefix,
			// which can not be reuse in other API(Open/Read) directly.
			// so we use TrimPrefix to filter Prefix for next Open/Read.
			trimmedKey := strings.TrimPrefix(r.Key, cliPrefix)
			// trim the prefix '/' to ensure that the path returned is consistent with the local storage
			trimmedKey = strings.TrimPrefix(trimmedKey, "/")
			itemSize := r.Size

			// filter out s3's empty directory items
			if itemSize <= 0 && strings.HasSuffix(trimmedKey, "/") {
				log.Info("skip empty directory which cannot be opened", zap.String("key", trimmedKey))
				continue
			}
			if err = fn(trimmedKey, itemSize); err != nil {
				return errors.Trace(err)
			}
		}