Repository: nspcc-dev/neofs-s3-gw
Pull request #1166: Store object tags as attributes along with a payload
Author: login: smallhive
Created: 2025-05-13T07:19:43Z | Merged: 2025-05-19T13:45:52Z

Closes #1153.

It would help to make tag based searches in the future.

Linked issue #1153: Extend tag storage scheme
## Is your feature request related to a problem? Please describe.
I'm always frustrated when I see out storage scheme is not future-proof. S3 tags are not very useful on their own, setting/getting them is not the real purpose. They become useful in various filters, mostly lifecycle, but also policy, replication and notifications. We support none of these functions, but for them to be possible we need tag object scheme to be flexible enough already.

## Describe the solution you'd like
1. Store tags in `S3-Meta-Tag-XXX: YYY` attributes along with JSON-based body when using separate tag objects. This makes it possible to do tag-based searches.
2. Do not create additional objects when tags are provided along with the object itself (store in attributes only). This optimizes out tag retrieval in most cases, tags are more likely to be provided at creation stage rather than separately.

## Describe alternatives you've considered
Keeping things as is.

## Additional context
https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html
https://docs.aws.amazon.com/AmazonS3/latest/userguide/tagging-and-policies.html

Diff:
diff --git a/api/handler/get.go b/api/handler/get.go
index c47ed445d..4621c5bc7 100644
--- a/api/handler/get.go
+++ b/api/handler/get.go
@@ -167,6 +167,15 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// There are no tags in separate objects. Try to get tags from the object headers.
+	if len(comprehensiveObjectInfo.TagSet) == 0 && objectWithPayloadReader.ObjectInfo != nil {
+		for k, v := range objectWithPayloadReader.ObjectInfo.Headers {
+			if strings.HasPrefix(k, s3headers.NeoFSSystemMetadataTagPrefix) {
+				comprehensiveObjectInfo.TagSet[strings.TrimPrefix(k, s3headers.NeoFSSystemMetadataTagPrefix)] = v
+			}
+		}
+	}
+
 	info := objectWithPayloadReader.ObjectInfo
 
 	if err = checkPreconditions(info, conditional); err != nil {
diff --git a/api/handler/put.go b/api/handler/put.go
index 6333d200c..5e9eebb94 100644
--- a/api/handler/put.go
+++ b/api/handler/put.go
@@ -271,6 +271,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 		Header:       metadata,
 		Encryption:   encryptionParams,
 		CopiesNumber: copiesNumber,
+		Tags:         tagSet,
 	}
 
 	params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
@@ -311,27 +312,6 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if tagSet != nil {
-		tagPrm := &layer.PutObjectTaggingParams{
-			ObjectVersion: &layer.ObjectVersion{
-				BktInfo:    bktInfo,
-				ObjectName: objInfo.Name,
-				VersionID:  objInfo.VersionID(),
-			},
-			TagSet:       tagSet,
-			CopiesNumber: h.cfg.CopiesNumber,
-		}
-
-		if !settings.VersioningEnabled() {
-			tagPrm.ObjectVersion.VersionID = ""
-		}
-
-		if err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
-			h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
-			return
-		}
-	}
-
 	if newEaclTable != nil {
 		p := &layer.PutBucketACLParams{
 			BktInfo:      bktInfo,
@@ -502,6 +482,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
 		Reader:  contentReader,
 		Size:    size,
 		Header:  metadata,
+		Tags:    tagSet,
 	}
 
 	extendedObjInfo, err := h.obj.PutObject(r.Context(), params)
@@ -538,22 +519,6 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if tagSet != nil {
-		tagPrm := &layer.PutObjectTaggingParams{
-			ObjectVersion: &layer.ObjectVersion{
-				BktInfo:    bktInfo,
-				ObjectName: objInfo.Name,
-				VersionID:  objInfo.VersionID(),
-			},
-			CopiesNumber: h.cfg.CopiesNumber,
-		}
-
-		if err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
-			h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
-			return
-		}
-	}
-
 	if newEaclTable != nil {
 		p := &layer.PutBucketACLParams{
 			BktInfo:      bktInfo,
diff --git a/api/handler/tagging.go b/api/handler/tagging.go
index 0065eda92..07341502e 100644
--- a/api/handler/tagging.go
+++ b/api/handler/tagging.go
@@ -184,7 +184,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
 		p.VersionID = ei.EncodeToString()
 	}
 
-	if err = h.obj.DeleteObjectTagging(r.Context(), p); err != nil {
+	if err = h.obj.DeleteObjectTagging(r.Context(), p, h.cfg.CopiesNumber); err != nil {
 		h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
 		return
 	}
diff --git a/api/layer/layer.go b/api/layer/layer.go
index f5da8a930..39ff57cda 100644
--- a/api/layer/layer.go
+++ b/api/layer/layer.go
@@ -120,6 +120,7 @@ type (
 		Lock         *data.ObjectLock
 		Encryption   encryption.Params
 		CopiesNumber uint32
+		Tags         map[string]string
 	}
 
 	DeleteObjectParams struct {
@@ -251,7 +252,7 @@ type (
 
 		GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error)
 		PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) error
-		DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error
+		DeleteObjectTagging(ctx context.Context, p *ObjectVersion, copiesNumber uint32) error
 
 		PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error)
 
@@ -662,10 +663,11 @@ func (n *layer) ComprehensiveObjectInfo(ctx context.Context, p *HeadObjectParams
 		owner         = n.Owner(ctx)
 		versions      []allVersionsSearchResult
 
-		tagSet         map[string]string
+		tagSet         = make(map[string]string)
 		lockInfo       *data.LockInfo
 		isEmptyVersion = len(p.VersionID) == 0
 		isNullVersion  = p.VersionID == data.UnversionedObjectVersionID
+		header         *object.Object
 	)
 
 	if isEmptyVersion || isNullVersion {
@@ -698,7 +700,7 @@ func (n *layer) ComprehensiveObjectInfo(ctx context.Context, p *HeadObjectParams
 			return nil, s3errors.GetAPIError(s3errors.ErrNoSuchVersion)
 		}
 
-		if _, err = n.objectHead(ctx, p.BktInfo, id); err != nil {
+		if header, err = n.objectHead(ctx, p.BktInfo, id); err != nil {
 			var errNotFound *apistatus.ObjectNotFound
 
 			if errors.As(err, &errNotFound) {
@@ -708,6 +710,12 @@ func (n *layer) ComprehensiveObjectInfo(ctx context.Context, p *HeadObjectParams
 			return nil, fmt.Errorf("head version %s: %w", p.VersionID, err)
 		}
 
+		for _, attr := range header.Attributes() {
+			if strings.HasPrefix(attr.Key(), s3headers.NeoFSSystemMetadataTagPrefix) {
+				tagSet[strings.TrimPrefix(attr.Key(), s3headers.NeoFSSystemMetadataTagPrefix)] = attr.Value()
+			}
+		}
+
 		tagsObjectOID, lockInfo, err = n.searchTagsAndLocksInNeoFS(ctx, p.BktInfo, owner, p.Object, p.VersionID)
 		if err != nil {
 			return nil, err
diff --git a/api/layer/object.go b/api/layer/object.go
index 790d855d9..77958610e 100644
--- a/api/layer/object.go
+++ b/api/layer/object.go
@@ -265,6 +265,10 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
 		}
 	}
 
+	for k, v := range p.Tags {
+		p.Header[s3headers.NeoFSSystemMetadataTagPrefix+k] = v
+	}
+
 	prm := PrmObjectCreate{
 		Container:    p.BktInfo.CID,
 		Creator:      owner,
diff --git a/api/layer/tagging.go b/api/layer/tagging.go
index fd898b4a4..1d0d9ba45 100644
--- a/api/layer/tagging.go
+++ b/api/layer/tagging.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"slices"
 	"strconv"
+	"strings"
 
 	"github.com/nspcc-dev/neofs-s3-gw/api/data"
 	"github.com/nspcc-dev/neofs-s3-gw/api/s3errors"
@@ -35,6 +36,7 @@ type taggingSearchResult struct {
 	ID                oid.ID
 	FilePath          string
 	CreationTimestamp int64
+	IsOriginalObject  bool
 }
 
 func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) error {
@@ -49,7 +51,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams)
 		CreationTime: TimeNow(ctx),
 		CopiesNumber: p.CopiesNumber,
 		Filepath:     p.ObjectVersion.ObjectName,
-		Attributes:   make(map[string]string, 2),
+		Attributes:   make(map[string]string, 3+len(p.TagSet)),
 		Payload:      bytes.NewBuffer(payload),
 		PayloadSize:  uint64(len(payload)),
 	}
@@ -60,6 +62,9 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams)
 	}
 
 	prm.Attributes[s3headers.MetaType] = s3headers.TypeTags
+	for k, v := range p.TagSet {
+		prm.Attributes[s3headers.NeoFSSystemMetadataTagPrefix+k] = v
+	}
 
 	if _, _, err = n.objectPutAndHash(ctx, prm, p.ObjectVersion.BktInfo); err != nil {
 		return fmt.Errorf("create tagging object: %w", err)
@@ -85,6 +90,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 		returningAttributes = []string{
 			object.AttributeFilePath,
 			object.AttributeTimestamp,
+			s3headers.MetaType,
 		}
 
 		opts client.SearchObjectsOptions
@@ -95,7 +101,6 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 	}
 
 	filters.AddFilter(object.AttributeFilePath, p.ObjectVersion.ObjectName, object.MatchStringEqual)
-	filters.AddFilter(s3headers.MetaType, s3headers.TypeTags, object.MatchStringEqual)
 	filters.AddTypeFilter(object.MatchStringEqual, object.TypeRegular)
 	if p.ObjectVersion.VersionID != "" {
 		filters.AddFilter(s3headers.AttributeObjectVersion, p.ObjectVersion.VersionID, object.MatchStringEqual)
@@ -111,10 +116,30 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 	}
 
 	if len(searchResultItems) == 0 {
+		// Objects inside versioned container don't have s3headers.AttributeObjectVersion attribute.
+		// This case means there are no separate attbute meta objects, let's try to get attributes from object itself.
+		if p.ObjectVersion.VersionID != "" {
+			var objID oid.ID
+			if err = objID.DecodeString(p.ObjectVersion.VersionID); err != nil {
+				return "", nil, fmt.Errorf("parse object version %s into oid: %w", p.ObjectVersion.VersionID, err)
+			}
+
+			tags, err := n.getTagsFromOriginalObject(ctx, p.ObjectVersion.BktInfo, objID)
+			if err != nil {
+				return "", nil, fmt.Errorf("get tags by oid: %w", err)
+			}
+
+			return p.ObjectVersion.VersionID, tags, nil
+		}
+
 		return "", nil, nil
 	}
 
-	var searchResults = make([]taggingSearchResult, 0, len(searchResultItems))
+	var (
+		tags                        map[string]string
+		metaObjectSearchResults     = make([]taggingSearchResult, 0, len(searchResultItems))
+		originalObjectSearchResults = make([]taggingSearchResult, 0, 1)
+	)
 
 	for _, item := range searchResultItems {
 		if len(item.Attributes) != len(returningAttributes) {
@@ -126,6 +151,16 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 			FilePath: item.Attributes[0],
 		}
 
+		switch item.Attributes[2] {
+		case "":
+			psr.IsOriginalObject = true
+		case s3headers.TypeTags:
+			// is a tag meta object.
+		default:
+			// skip other meta types.
+			continue
+		}
+
 		if item.Attributes[1] != "" {
 			psr.CreationTimestamp, err = strconv.ParseInt(item.Attributes[1], 10, 64)
 			if err != nil {
@@ -133,7 +168,11 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 			}
 		}
 
-		searchResults = append(searchResults, psr)
+		if psr.IsOriginalObject {
+			originalObjectSearchResults = append(originalObjectSearchResults, psr)
+		} else {
+			metaObjectSearchResults = append(metaObjectSearchResults, psr)
+		}
 	}
 
 	sortFunc := func(a, b taggingSearchResult) int {
@@ -145,16 +184,43 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 		return bytes.Compare(b.ID[:], a.ID[:]) // reverse order.
 	}
 
-	slices.SortFunc(searchResults, sortFunc)
+	// There are not extra meta objects with tags.
+	if len(metaObjectSearchResults) == 0 {
+		slices.SortFunc(originalObjectSearchResults, sortFunc)
 
-	tags, err := n.getTagsByOID(ctx, p.ObjectVersion.BktInfo, searchResults[0].ID)
-	if err != nil {
-		return "", nil, err
+		tags, err = n.getTagsFromOriginalObject(ctx, p.ObjectVersion.BktInfo, originalObjectSearchResults[0].ID)
+		if err != nil {
+			return "", nil, err
+		}
+	} else {
+		slices.SortFunc(metaObjectSearchResults, sortFunc)
+
+		tags, err = n.getTagsByOID(ctx, p.ObjectVersion.BktInfo, metaObjectSearchResults[0].ID)
+		if err != nil {
+			return "", nil, err
+		}
 	}
 
 	return p.ObjectVersion.VersionID, tags, nil
 }
 
+func (n *layer) getTagsFromOriginalObject(ctx context.Context, bktInfo *data.BucketInfo, id oid.ID) (map[string]string, error) {
+	var tagSet = make(map[string]string)
+
+	header, err := n.objectHead(ctx, bktInfo, id)
+	if err != nil {
+		return nil, fmt.Errorf("get object head: %w", err)
+	}
+
+	for _, attr := range header.Attributes() {
+		if strings.HasPrefix(attr.Key(), s3headers.NeoFSSystemMetadataTagPrefix) {
+			tagSet[strings.TrimPrefix(attr.Key(), s3headers.NeoFSSystemMetadataTagPrefix)] = attr.Value()
+		}
+	}
+
+	return tagSet, nil
+}
+
 func (n *layer) getTagsByOID(ctx context.Context, bktInfo *data.BucketInfo, id oid.ID) (map[string]string, error) {
 	lastObj, err := n.objectGet(ctx, bktInfo, id)
 	if err != nil {
@@ -176,7 +242,7 @@ func (n *layer) getTagsByOID(ctx context.Context, bktInfo *data.BucketInfo, id o
 	return tags, nil
 }
 
-func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error {
+func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion, copiesNumber uint32) error {
 	fs := make(object.SearchFilters, 0, 4)
 	fs.AddFilter(object.AttributeFilePath, p.ObjectName, object.MatchStringEqual)
 	fs.AddFilter(s3headers.MetaType, s3headers.TypeTags, object.MatchStringEqual)
@@ -211,7 +277,14 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error
 
 	n.cache.DeleteTagging(objectTaggingCacheKey(p))
 
-	return nil
+	// Put an empty tag list to override tags inside the object itself.
+	putObjectTaggingParams := PutObjectTaggingParams{
+		ObjectVersion: p,
+		TagSet:        make(map[string]string),
+		CopiesNumber:  copiesNumber,
+	}
+
+	return n.PutObjectTagging(ctx, &putObjectTaggingParams)
 }
 
 func (n *layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
diff --git a/api/s3headers/headers.go b/api/s3headers/headers.go
index 248a4d29b..3424a4c48 100644
--- a/api/s3headers/headers.go
+++ b/api/s3headers/headers.go
@@ -79,6 +79,9 @@ const (
 	AttributeVersioningState     = NeoFSSystemMetadataPrefix + "VersioningState"
 	AttributeDeleteMarker        = NeoFSSystemMetadataPrefix + "DeleteMarker"
 	UploadCompletedParts         = NeoFSSystemMetadataPrefix + "Completed-Parts"
+
+	// Result: S3-Meta-Tag-.
+	NeoFSSystemMetadataTagPrefix = NeoFSSystemMetadataPrefix + "Tag-"
 )
 
 const (


Reviews:
user: login: roman-khimov
body: How about tag deletion?
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-14T14:29:12Z
user: login: smallhive
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T05:07:38Z
user: login: smallhive
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T08:17:20Z
user: login: roman-khimov
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-19T13:45:19Z

Review comments:
diff_hunk: @@ -158,6 +160,55 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 	return p.ObjectVersion.VersionID, tags, nil
 }
 
+func (n *layer) getTagsFromOriginalObject(ctx context.Context, bktInfo *data.BucketInfo, name, version string) (map[string]string, error) {
+	var (
+		id             oid.ID
+		isEmptyVersion = len(version) == 0
+		isNullVersion  = version == data.UnversionedObjectVersionID
+		tagSet         = make(map[string]string)
+	)
+
+	if isEmptyVersion || isNullVersion {
+		versions, _, _, err := n.comprehensiveSearchAllVersionsInNeoFS(ctx, bktInfo, n.Owner(ctx), name, isNullVersion)
path: api/layer/tagging.go
user: login: roman-khimov
body: Additional searches can and should be avoided.
created_at: 2025-05-14T14:28:54Z
author_association: MEMBER
diff_hunk: @@ -158,6 +160,55 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 	return p.ObjectVersion.VersionID, tags, nil
 }
 
+func (n *layer) getTagsFromOriginalObject(ctx context.Context, bktInfo *data.BucketInfo, name, version string) (map[string]string, error) {
+	var (
+		id             oid.ID
+		isEmptyVersion = len(version) == 0
+		isNullVersion  = version == data.UnversionedObjectVersionID
+		tagSet         = make(map[string]string)
+	)
+
+	if isEmptyVersion || isNullVersion {
+		versions, _, _, err := n.comprehensiveSearchAllVersionsInNeoFS(ctx, bktInfo, n.Owner(ctx), name, isNullVersion)
path: api/layer/tagging.go
user: login: smallhive
body: Do you suggest searching all object versions + all meta objects for this version inside `GetObjectTagging`. Then filter out the required data in the code? Is it cheaper than two searches?
created_at: 2025-05-15T05:07:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -158,6 +160,55 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
 	return p.ObjectVersion.VersionID, tags, nil
 }
 
+func (n *layer) getTagsFromOriginalObject(ctx context.Context, bktInfo *data.BucketInfo, name, version string) (map[string]string, error) {
+	var (
+		id             oid.ID
+		isEmptyVersion = len(version) == 0
+		isNullVersion  = version == data.UnversionedObjectVersionID
+		tagSet         = make(map[string]string)
+	)
+
+	if isEmptyVersion || isNullVersion {
+		versions, _, _, err := n.comprehensiveSearchAllVersionsInNeoFS(ctx, bktInfo, n.Owner(ctx), name, isNullVersion)
path: api/layer/tagging.go
user: login: smallhive
body: Reworked
created_at: 2025-05-16T08:17:20Z
author_association: CONTRIBUTOR

PR comments:
user: login: smallhive
created_at: 2025-05-15T05:08:16Z
body: > How about tag deletion?

It is a responsibility of #1158 
author_association: CONTRIBUTOR
user: login: roman-khimov
created_at: 2025-05-15T14:55:24Z
body: I mean deleting tags of an object that had them set on put.
author_association: MEMBER
user: login: smallhive
created_at: 2025-05-16T08:20:56Z
body: I consider you mean [this](https://github.com/nspcc-dev/neofs-s3-gw/pull/1166/files#diff-68e5a2a921a951a39c819d0470049094a5b68135095040955e0633319c785f78R281) about tag removing.
To handle tag removal from the object itself, we store the tags' meta object with empty tags. Any tags meta object has priority against tags inside object attributes
author_association: CONTRIBUTOR