Repository: googleforgames/open-saves
Pull request #399 (2022-10-12 06:27:38)

fix!: remove indexing for timestamps
**What type of PR is this?**
> /kind fix

**What this PR does / Why we need it**:
Removes indexing for all timestamps since it breaks [best practices](https://cloud.google.com/datastore/docs/best-practices#:~:text=Do%20not%20index,key%20range%20below.) around monotonically increasing values.

**Which issue(s) this PR fixes**:

**Special notes for your reviewer**:
This requires changes on the Open Saves collector, which previously depended on the use of the Timestamps fields to select entities older than a certain time. Now, we fetch all blobs/chunks with a status and only selectively delete.

This is also a breaking change to `QueryRecords` since we will no longer support querying by ascending/descending timestamps.

hongalex: The tests that are failing are only part of the collector and they are failing on main as well.
hongalex: Fixed the failing collector tests. PTAL @loranger2k 


Patch:
diff --git a/deploy/datastore/index.yaml b/deploy/datastore/index.yaml
index b5ab72b..87a37a2 100644
--- a/deploy/datastore/index.yaml
+++ b/deploy/datastore/index.yaml
@@ -1,36 +1,17 @@
 indexes:
- 
-- kind: blob
-  properties:
-  - name: Status
-    direction: asc
-  - name: Timestamps.UpdatedAt
-    direction: asc
- 
-- kind: chunk
-  properties:
-  - name: Status
-    direction: asc
-  - name: Timestamps.UpdatedAt
-    direction: asc
+  - kind: record
+    ancestor: yes
+    properties:
+      - name: Properties.prop1
+        direction: asc
 
-- kind: record
-  ancestor: yes
-  properties:
-  - name: Properties.prop1
-    direction: asc 
+  - kind: record
+    ancestor: yes
+    properties:
+      - name: Properties.prop1
 
-- kind: record
-  ancestor: yes
-  properties:
-  - name: Properties.prop1
-
-- kind: record
-  ancestor: yes
-  properties:
-  - name: Timestamps.CreatedAt
-
-- kind: record
-  ancestor: yes
-  properties:
-  - name: Timestamps.UpdatedAt
+  - kind: record
+    ancestor: yes
+    properties:
+      - name: Properties.prop1
+        direction: desc
diff --git a/go.mod b/go.mod
index 7075ef3..422e335 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/alicebob/miniredis/v2 v2.22.0
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/golang/mock v1.6.0
+	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
 	github.com/pseudomuto/protoc-gen-doc v1.5.1
 	github.com/sirupsen/logrus v1.9.0
@@ -38,7 +39,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
diff --git a/internal/app/collector/collector.go b/internal/app/collector/collector.go
index aaa6c1f..1538bfd 100644
--- a/internal/app/collector/collector.go
+++ b/internal/app/collector/collector.go
@@ -172,7 +172,7 @@ func (c *Collector) deleteBlob(ctx context.Context, blob *blobref.BlobRef) {
 
 func (c *Collector) deleteMatchingBlobRefs(ctx context.Context, status blobref.Status, olderThan time.Time) error {
 	log.Infof("Garbage collecting BlobRef objects with status = %v, and older than %v", status, olderThan)
-	cursor, err := c.metaDB.ListBlobRefsByStatus(ctx, status, olderThan)
+	cursor, err := c.metaDB.ListBlobRefsByStatus(ctx, status)
 	if err != nil {
 		log.Fatalf("ListBlobRefsByStatus returned error: %v", err)
 		return err
@@ -186,14 +186,16 @@ func (c *Collector) deleteMatchingBlobRefs(ctx context.Context, status blobref.S
 			log.Errorf("cursor.Next() returned error: %v", err)
 			break
 		}
-		c.deleteBlob(ctx, blob)
+		if blob.Timestamps.UpdatedAt.Before(olderThan) {
+			c.deleteBlob(ctx, blob)
+		}
 	}
 	return nil
 }
 
 func (c *Collector) deleteMatchingChunkRefs(ctx context.Context, status blobref.Status, olderThan time.Time) error {
-	log.Infof("Garbage collecting ChunkRef objects with status = %v, older than %v", status, olderThan)
-	cursor := c.metaDB.ListChunkRefsByStatus(ctx, status, olderThan)
+	log.Infof("Garbage collecting ChunkRef objects with status = %v, and older than %v", status, olderThan)
+	cursor := c.metaDB.ListChunkRefsByStatus(ctx, status)
 	for {
 		chunk, err := cursor.Next()
 		if err == iterator.Done {
@@ -203,12 +205,14 @@ func (c *Collector) deleteMatchingChunkRefs(ctx context.Context, status blobref.
 			log.Errorf("cursor.Next() return error: %v", err)
 			return err
 		}
-		if err := c.deleteChunk(ctx, chunk); err != nil {
-			log.Errorf("deleteChunk failed for chunk (%v): %v", chunk.Key, err)
-			continue
-		}
-		if err := c.metaDB.DeleteChunkRef(ctx, chunk.BlobRef, chunk.Key); err != nil {
-			log.Errorf("DeleteChunkRef failed for chunk (%v): %v", chunk.Key, err)
+		if chunk.Timestamps.UpdatedAt.Before(olderThan) {
+			if err := c.deleteChunk(ctx, chunk); err != nil {
+				log.Errorf("deleteChunk failed for chunk (%v): %v", chunk.Key, err)
+				continue
+			}
+			if err := c.metaDB.DeleteChunkRef(ctx, chunk.BlobRef, chunk.Key); err != nil {
+				log.Errorf("DeleteChunkRef failed for chunk (%v): %v", chunk.Key, err)
+			}
 		}
 	}
 	return nil
diff --git a/internal/pkg/metadb/metadb.go b/internal/pkg/metadb/metadb.go
index 72dbdf1..31c01c7 100644
--- a/internal/pkg/metadb/metadb.go
+++ b/internal/pkg/metadb/metadb.go
@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
-	"time"
 
 	ds "cloud.google.com/go/datastore"
 	"github.com/google/uuid"
@@ -55,10 +54,10 @@ var (
 // The methods return gRPC error codes. Here are some common error codes
 // returned. For additional details, please look at the method help.
 // Common errors:
-//	- NotFound: entity or object is not found
-//	- Aborted: transaction is aborted
-//	- InvalidArgument: key or value provided is not valid
-//	- Internal: internal unrecoverable error
+//   - NotFound: entity or object is not found
+//   - Aborted: transaction is aborted
+//   - InvalidArgument: key or value provided is not valid
+//   - Internal: internal unrecoverable error
 type MetaDB struct {
 	// Datastore namespace for multi-tenancy
 	Namespace string
@@ -417,7 +416,7 @@ func (m *MetaDB) UpdateBlobRef(ctx context.Context, blob *blobref.BlobRef) (*blo
 
 // GetBlobRef returns a BlobRef object specified by the key.
 // Returns errors:
-//	- NotFound: the object is not found.
+//   - NotFound: the object is not found.
 func (m *MetaDB) GetBlobRef(ctx context.Context, key uuid.UUID) (*blobref.BlobRef, error) {
 	return m.getBlobRef(ctx, nil, key)
 }
@@ -434,8 +433,8 @@ func (m *MetaDB) getCurrentBlobRef(ctx context.Context, tx *ds.Transaction, stor
 
 // GetCurrentBlobRef gets a BlobRef object associated with a record.
 // Returned errors:
-// 	- NotFound: the record is not found.
-// 	- FailedPrecondition: the record doesn't have a blob.
+//   - NotFound: the record is not found.
+//   - FailedPrecondition: the record doesn't have a blob.
 func (m *MetaDB) GetCurrentBlobRef(ctx context.Context, storeKey, recordKey string) (*blobref.BlobRef, error) {
 	var blob *blobref.BlobRef
 	_, err := m.client.RunInTransaction(ctx, func(tx *ds.Transaction) error {
@@ -480,8 +479,8 @@ func (m *MetaDB) chunkObjectsSizeSum(ctx context.Context, tx *ds.Transaction, bl
 // PromoteBlobRefToCurrent promotes the provided BlobRef object as a current
 // external blob reference.
 // Returned errors:
-//	- NotFound: the specified record or the blobref was not found
-//  - Internal: BlobRef status transition error
+//   - NotFound: the specified record or the blobref was not found
+//   - Internal: BlobRef status transition error
 func (m *MetaDB) PromoteBlobRefToCurrent(ctx context.Context, blob *blobref.BlobRef) (*record.Record, *blobref.BlobRef, error) {
 	record := new(record.Record)
 	_, err := m.client.RunInTransaction(ctx, func(tx *ds.Transaction) error {
@@ -550,9 +549,9 @@ func (m *MetaDB) PromoteBlobRefToCurrent(ctx context.Context, blob *blobref.Blob
 // storeKey and recordKey. It also changes the status of the blob object to
 // BlobRefStatusPendingDeletion.
 // Returned errors:
-//	- NotFound: the specified record or the blobref was not found
-//	- FailedPrecondition: the record doesn't have an external blob
-//  - Internal: BlobRef status transition error
+//   - NotFound: the specified record or the blobref was not found
+//   - FailedPrecondition: the record doesn't have an external blob
+//   - Internal: BlobRef status transition error
 func (m *MetaDB) RemoveBlobFromRecord(ctx context.Context, storeKey string, recordKey string) (*record.Record, *blobref.BlobRef, error) {
 	rkey := m.createRecordKey(storeKey, recordKey)
 	blob := new(blobref.BlobRef)
@@ -629,8 +628,8 @@ func (m *MetaDB) deleteChildChunkRefs(ctx context.Context, tx *ds.Transaction, b
 
 // DeleteBlobRef deletes the BlobRef object from the database.
 // Returned errors:
-//	- NotFound: the blobref object is not found
-//	- FailedPrecondition: the blobref status is Ready and can't be deleted
+//   - NotFound: the blobref object is not found
+//   - FailedPrecondition: the blobref status is Ready and can't be deleted
 func (m *MetaDB) DeleteBlobRef(ctx context.Context, key uuid.UUID) error {
 	_, err := m.client.RunInTransaction(ctx, func(tx *ds.Transaction) error {
 		blob, err := m.getBlobRef(ctx, tx, key)
@@ -652,8 +651,8 @@ func (m *MetaDB) DeleteBlobRef(ctx context.Context, key uuid.UUID) error {
 
 // DeleteChunkRef deletes the ChunkRef object from the database.
 // Returned errors:
-//	- NotFound: the chunkref object is not found.
-//	- FailedPrecondition: the chunkref status is Ready and can't be deleted.
+//   - NotFound: the chunkref object is not found.
+//   - FailedPrecondition: the chunkref status is Ready and can't be deleted.
 func (m *MetaDB) DeleteChunkRef(ctx context.Context, blobKey, key uuid.UUID) error {
 	_, err := m.client.RunInTransaction(ctx, func(tx *ds.Transaction) error {
 		var chunk chunkref.ChunkRef
@@ -669,19 +668,17 @@ func (m *MetaDB) DeleteChunkRef(ctx context.Context, blobKey, key uuid.UUID) err
 }
 
 // ListBlobRefsByStatus returns a cursor that iterates over BlobRefs
-// where Status = status and UpdatedAt < olderThan.
-func (m *MetaDB) ListBlobRefsByStatus(ctx context.Context, status blobref.Status, olderThan time.Time) (*blobref.BlobRefCursor, error) {
-	query := m.newQuery(blobKind).Filter("Status = ", int(status)).
-		Filter("Timestamps.UpdatedAt <", olderThan)
+// where Status = status.
+func (m *MetaDB) ListBlobRefsByStatus(ctx context.Context, status blobref.Status) (*blobref.BlobRefCursor, error) {
+	query := m.newQuery(blobKind).Filter("Status = ", int(status))
 	iter := blobref.NewCursor(m.client.Run(ctx, query))
 	return iter, nil
 }
 
 // ListChunkRefsByStatus returns a cursor that iterates over ChunkRefs
-// where Status = status and UpdatedAt < olderThan.
-func (m *MetaDB) ListChunkRefsByStatus(ctx context.Context, status blobref.Status, olderThan time.Time) *chunkref.ChunkRefCursor {
-	query := m.newQuery(chunkKind).Filter("Status = ", int(status)).
-		Filter("Timestamps.UpdatedAt <", olderThan)
+// where Status = status.
+func (m *MetaDB) ListChunkRefsByStatus(ctx context.Context, status blobref.Status) *chunkref.ChunkRefCursor {
+	query := m.newQuery(chunkKind).Filter("Status = ", int(status))
 	return chunkref.NewCursor(m.client.Run(ctx, query))
 }
 
diff --git a/internal/pkg/metadb/record/record.go b/internal/pkg/metadb/record/record.go
index 3485fd2..149ecf8 100644
--- a/internal/pkg/metadb/record/record.go
+++ b/internal/pkg/metadb/record/record.go
@@ -83,13 +83,13 @@ func (r *Record) Save() ([]datastore.Property, error) {
 // Load implements the Datastore PropertyLoadSaver interface and converts Datastore
 // properties to corresponding struct fields.
 func (r *Record) Load(ps []datastore.Property) error {
-        // Added for backward compatibility.
+	// Added for backward compatibility.
 	for i, p := range ps {
-                if p.Name == "NumberOfChunks" {
-                        ps[i].Name = "ChunkCount"
+		if p.Name == "NumberOfChunks" {
+			ps[i].Name = "ChunkCount"
 			break
-                }
-        }
+		}
+	}
 	externalBlob, ps, err := timestamps.LoadUUID(ps, externalBlobPropertyName)
 	if err != nil {
 		return err
diff --git a/internal/pkg/metadb/timestamps/timestamps.go b/internal/pkg/metadb/timestamps/timestamps.go
index 3919e3f..ef3f1a9 100644
--- a/internal/pkg/metadb/timestamps/timestamps.go
+++ b/internal/pkg/metadb/timestamps/timestamps.go
@@ -37,10 +37,10 @@ const (
 type Timestamps struct {
 	// CreatedAt is the timestamp of the record creation time
 	// Automatically set by MetaDB
-	CreatedAt time.Time
+	CreatedAt time.Time `datastore:",noindex"`
 	// UpdatedAt is the timestamp of the last modification time
 	// Automatically set and managed by MetaDB
-	UpdatedAt time.Time
+	UpdatedAt time.Time `datastore:",noindex"`
 	// Signature is a UUID that is randomly created each time the record is updated
 	// Automatically set and managed by MetaDB
 	Signature uuid.UUID `datastore:"-"`


Test patch:
diff --git a/internal/app/collector/collector_test.go b/internal/app/collector/collector_test.go
index 9530593..6915599 100644
--- a/internal/app/collector/collector_test.go
+++ b/internal/app/collector/collector_test.go
@@ -155,8 +155,8 @@ func TestCollector_DeletesBlobs(t *testing.T) {
 	// 4 is still initializing
 	for i := 0; i < blobRefCount; i++ {
 		blobRef := blobref.NewBlobRef(0, store.Key, record.Key)
-		blobRef.Timestamps.CreatedAt = collector.cfg.Before
-		blobRef.Timestamps.UpdatedAt = collector.cfg.Before
+		blobRef.Timestamps.CreatedAt = time.Now()
+		blobRef.Timestamps.UpdatedAt = time.Now()
 		blobRefs = append(blobRefs, blobRef)
 	}
 	blobRefs[0].MarkForDeletion()
@@ -242,8 +242,8 @@ func TestCollector_DeletesChunkedBlobs(t *testing.T) {
 	// 4 is still initializing
 	for i := 0; i < chunkRefCount; i++ {
 		chunk := chunkref.New(blob.Key, int32(i))
-		chunk.Timestamps.CreatedAt = collector.cfg.Before
-		chunk.Timestamps.UpdatedAt = collector.cfg.Before
+		chunk.Timestamps.CreatedAt = time.Now()
+		chunk.Timestamps.UpdatedAt = time.Now()
 		chunks = append(chunks, chunk)
 	}
 	chunks[0].MarkForDeletion()
diff --git a/internal/app/server/open_saves_test.go b/internal/app/server/open_saves_test.go
index 7c20342..341e64a 100644
--- a/internal/app/server/open_saves_test.go
+++ b/internal/app/server/open_saves_test.go
@@ -1164,21 +1164,6 @@ func TestOpenSaves_QueryRecords_Order(t *testing.T) {
 	assert.Equal(t, resp.Records[0].Properties["prop1"].Value, intVal1)
 	assert.Equal(t, resp.Records[1].Properties["prop1"].Value, intVal2)
 
-	queryReq = &pb.QueryRecordsRequest{
-		StoreKey: storeKey,
-		SortOrders: []*pb.SortOrder{
-			{
-				Property:  pb.SortOrder_UPDATED_AT,
-				Direction: pb.SortOrder_ASC,
-			},
-		},
-	}
-	resp, err = client.QueryRecords(ctx, queryReq)
-	// These records are created at the same time so no way of verifying the order.
-	// Just make sure there's no error returned by this query.
-	require.NoError(t, err)
-	require.Equal(t, 2, len(resp.Records))
-
 	// Test errors
 	queryReq = &pb.QueryRecordsRequest{
 		StoreKey: storeKey,
@@ -1191,18 +1176,6 @@ func TestOpenSaves_QueryRecords_Order(t *testing.T) {
 	}
 	_, err = client.QueryRecords(ctx, queryReq)
 	assert.Equal(t, codes.InvalidArgument, status.Code(err))
-
-	queryReq = &pb.QueryRecordsRequest{
-		StoreKey: storeKey,
-		SortOrders: []*pb.SortOrder{
-			{
-				Property:  pb.SortOrder_CREATED_AT,
-				Direction: 3,
-			},
-		},
-	}
-	_, err = client.QueryRecords(ctx, queryReq)
-	assert.Equal(t, codes.InvalidArgument, status.Code(err))
 }
 
 func TestOpenSaves_QueryRecords_Limit(t *testing.T) {
diff --git a/internal/pkg/metadb/blobref/blobref_test.go b/internal/pkg/metadb/blobref/blobref_test.go
index 58c7cc1..31648f3 100644
--- a/internal/pkg/metadb/blobref/blobref_test.go
+++ b/internal/pkg/metadb/blobref/blobref_test.go
@@ -155,12 +155,14 @@ func TestBlobRef_Save(t *testing.T) {
 			Value: &datastore.Entity{
 				Properties: []datastore.Property{
 					{
-						Name:  "CreatedAt",
-						Value: time.Date(1992, 1, 15, 3, 15, 55, 0, time.UTC),
+						Name:    "CreatedAt",
+						Value:   time.Date(1992, 1, 15, 3, 15, 55, 0, time.UTC),
+						NoIndex: true,
 					},
 					{
-						Name:  "UpdatedAt",
-						Value: time.Date(1992, 11, 27, 1, 3, 11, 0, time.UTC),
+						Name:    "UpdatedAt",
+						Value:   time.Date(1992, 11, 27, 1, 3, 11, 0, time.UTC),
+						NoIndex: true,
 					},
 					{
 						Name:    "Signature",
diff --git a/internal/pkg/metadb/metadb_test.go b/internal/pkg/metadb/metadb_test.go
index 466cc4c..dca7052 100644
--- a/internal/pkg/metadb/metadb_test.go
+++ b/internal/pkg/metadb/metadb_test.go
@@ -845,7 +845,7 @@ func TestMetaDB_ListBlobsByStatus(t *testing.T) {
 		blobref.StatusPendingDeletion,
 		blobref.StatusPendingDeletion}
 	blobs := []*blobref.BlobRef{}
-	for i, s := range statuses {
+	for _, s := range statuses {
 		blob := &blobref.BlobRef{
 			Key:       uuid.New(),
 			Status:    s,
@@ -855,8 +855,6 @@ func TestMetaDB_ListBlobsByStatus(t *testing.T) {
 		blobs = append(blobs, blob)
 		setupTestBlobRef(ctx, t, metaDB, blob)
 
-		// Update the timestamps here because MetaDB automatically sets UpdatesAt
-		blob.Timestamps.UpdatedAt = time.Date(2000, 1, i, 0, 0, 0, 0, time.UTC)
 		bKey := blobRefKey(blob.Key)
 		if _, err := client.Put(ctx, bKey, blob); err != nil {
 			t.Fatalf("Failed to change UpdatedAt for %v: %v", blob.Key, err)
@@ -864,7 +862,7 @@ func TestMetaDB_ListBlobsByStatus(t *testing.T) {
 	}
 
 	// Should return iterator.Done and nil when not found
-	iter, err := metaDB.ListBlobRefsByStatus(ctx, blobref.StatusError, time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC))
+	iter, err := metaDB.ListBlobRefsByStatus(ctx, blobref.StatusUnknown)
 	assert.NoError(t, err)
 	if assert.NotNil(t, iter) {
 		b, err := iter.Next()
@@ -872,24 +870,16 @@ func TestMetaDB_ListBlobsByStatus(t *testing.T) {
 		assert.Nil(t, b)
 	}
 
-	iter, err = metaDB.ListBlobRefsByStatus(ctx, blobref.StatusPendingDeletion, time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC))
-	assert.NoError(t, err)
-	if assert.NotNil(t, iter) {
-		// Should return both of the PendingDeletion entries
-		got, err := iter.Next()
-		assert.NoError(t, err)
-		if diff := cmp.Diff(blobs[2], got, cmpopts.EquateEmpty()); diff != "" {
-			t.Errorf("Next() = (-want, +got):\n%s", diff)
-		}
-
-		got, err = iter.Next()
-		assert.NoError(t, err)
-		if diff := cmp.Diff(blobs[3], got, cmpopts.EquateEmpty()); diff != "" {
-			t.Errorf("Next() = (-want, +got):\n%s", diff)
+	iter, err = metaDB.ListBlobRefsByStatus(ctx, blobref.StatusPendingDeletion)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	got, err := iter.Next()
+	for err == nil {
+		if got.Status != blobref.StatusPendingDeletion {
+			t.Fatalf("got status %v, want %v", got.Status, blobref.StatusPendingDeletion)
 		}
 		got, err = iter.Next()
-		assert.Equal(t, iterator.Done, err)
-		assert.Nil(t, got)
 	}
 }
 
@@ -1239,50 +1229,37 @@ func TestMetaDB_ListChunkRefsByStatus(t *testing.T) {
 	}
 
 	testCase := []struct {
-		name      string
-		status    blobref.Status
-		olderThan time.Time
-		want      []*chunkref.ChunkRef
+		name   string
+		status blobref.Status
 	}{
 		{
 			"not found",
-			blobref.StatusError,
-			time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC),
-			[]*chunkref.ChunkRef{},
+			blobref.StatusUnknown,
 		},
 		{
 			"PendingDeletion",
 			blobref.StatusPendingDeletion,
-			time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC),
-			[]*chunkref.ChunkRef{chunks[2], chunks[3]},
 		},
 	}
 	for _, tc := range testCase {
 		t.Run(tc.name, func(t *testing.T) {
-			iter := metaDB.ListChunkRefsByStatus(ctx, tc.status, tc.olderThan)
+			iter := metaDB.ListChunkRefsByStatus(ctx, tc.status)
 			if iter == nil {
 				t.Fatalf("ListChunkRefsByStatus() = %v, want non-nil", iter)
 			}
-			got := []*chunkref.ChunkRef{}
-			for i := 0; i < len(tc.want); i++ {
-				c, err := iter.Next()
-				if err != nil {
-					t.Errorf("Next() failed: %v", err)
+			c, err := iter.Next()
+			for err == nil {
+				if c.Status != tc.status {
+					t.Fatalf("got status %v, want %v", c.Status, tc.status)
 				}
-				got = append(got, c)
+				c, err = iter.Next()
 			}
-			c, err := iter.Next()
 			if !errors.Is(err, iterator.Done) {
 				t.Errorf("Next() = %v, want = iterator.Done", err)
 			}
 			if c != nil {
 				t.Errorf("Next() = %v, want = nil", iter)
 			}
-			if diff := cmp.Diff(tc.want, got,
-				cmpopts.SortSlices(func(a, b *chunkref.ChunkRef) bool { return a.Number < b.Number }),
-			); diff != "" {
-				t.Errorf("ListChunkRefsByStatus() iterator = (-want, +got):\n%s", diff)
-			}
 		})
 	}
 }
@@ -1353,9 +1330,9 @@ func TestMetaDB_QueryRecords(t *testing.T) {
 		{
 			"Limit",
 			&pb.QueryRecordsRequest{
-				StoreKey:   stores[0].Key,
-				SortOrders: []*pb.SortOrder{{Property: pb.SortOrder_CREATED_AT}},
-				Limit:      1,
+				StoreKey: stores[0].Key,
+				Tags:     []string{"tag1"},
+				Limit:    1,
 			},
 			[]*record.Record{records[0]}, codes.OK,
 		},
diff --git a/internal/pkg/metadb/record/record_test.go b/internal/pkg/metadb/record/record_test.go
index 479b49b..32d62bb 100644
--- a/internal/pkg/metadb/record/record_test.go
+++ b/internal/pkg/metadb/record/record_test.go
@@ -124,12 +124,14 @@ func TestRecord_Save(t *testing.T) {
 				Value: &datastore.Entity{
 					Properties: []datastore.Property{
 						{
-							Name:  "CreatedAt",
-							Value: createdAt,
+							Name:    "CreatedAt",
+							Value:   createdAt,
+							NoIndex: true,
 						},
 						{
-							Name:  "UpdatedAt",
-							Value: updatedAt,
+							Name:    "UpdatedAt",
+							Value:   updatedAt,
+							NoIndex: true,
 						},
 						{
 							Name:    "Signature",
diff --git a/internal/pkg/metadb/timestamps/timestamps_test.go b/internal/pkg/metadb/timestamps/timestamps_test.go
index 9243310..9dc6259 100644
--- a/internal/pkg/metadb/timestamps/timestamps_test.go
+++ b/internal/pkg/metadb/timestamps/timestamps_test.go
@@ -119,12 +119,14 @@ func TestTimestamps_Save(t *testing.T) {
 	}
 	want := []datastore.Property{
 		{
-			Name:  "CreatedAt",
-			Value: ts.CreatedAt,
+			Name:    "CreatedAt",
+			Value:   ts.CreatedAt,
+			NoIndex: true,
 		},
 		{
-			Name:  "UpdatedAt",
-			Value: ts.UpdatedAt,
+			Name:    "UpdatedAt",
+			Value:   ts.UpdatedAt,
+			NoIndex: true,
 		},
 		{
 			Name:    "Signature",