Repository: weaviate/weaviate
Pull request #8173: Query the leader in the consumer semaphore to avoid racey state change
Author: login: tsmith023
Created: 2025-05-16T15:10:48Z | Merged: 2025-05-18T09:28:36Z

### What's being changed:

- Previously, if the op was produced into the channel in state A then mutated by the consumer to state B then the consumer would accept the stale op from the channel and re-process it
- Now, the consumer queries the leader for the most up-to-date state of an op and skips the op if the new state is different from the old state

### Review checklist

- [ ] Documentation has been updated, if necessary. Link to changed documentation:
- [ ] Chaos pipeline run or not necessary. Link to pipeline:
- [x] All new code is covered by tests where it is reasonable.
- [ ] Performance tests have been run or not necessary.

<!-- Uncomment the following section if this PR requires changes in related projects (e.g., documentation, client libraries).

GitHub actions will automatically create an issue in the corresponding repository for each checked box below. (See `.github/workflows/create-cross-functional-issues.yml`)

### Cross-functional impact

- [ ] This change requires public documentation (weaviate-io) to be updated. Check the box to automatically create a corresponding issue.
- Does it require a change in the client libraries? If yes, please check the boxes for the affected client libraries.
    - [ ] Python (weaviate-python-client)
    - [ ] JavaScript/TypeScript (typescript-client)
    - [ ] Go (weaviate-go-client)
    - [ ] Java (java-client)

-->



Diff:
diff --git a/adapters/repos/db/replication.go b/adapters/repos/db/replication.go
index 78a5b2c53e2..29cd986cd2a 100644
--- a/adapters/repos/db/replication.go
+++ b/adapters/repos/db/replication.go
@@ -369,9 +369,12 @@ func (i *Index) IncomingListFiles(ctx context.Context,
 	shardName string,
 ) ([]string, error) {
 	shard, release, err := i.GetShard(ctx, shardName)
-	if err != nil || shard == nil {
+	if err != nil {
 		return nil, fmt.Errorf("incoming list files get shard %s: %w", shardName, err)
 	}
+	if shard == nil {
+		return nil, fmt.Errorf("incoming list files get shard is nil: %s", shardName)
+	}
 	defer release()
 
 	sd := backup.ShardDescriptor{Name: shardName}
diff --git a/cluster/proto/api/message.pb.go b/cluster/proto/api/message.pb.go
index 09becabe847..cfbbb828782 100644
--- a/cluster/proto/api/message.pb.go
+++ b/cluster/proto/api/message.pb.go
@@ -56,9 +56,9 @@ const (
 	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE                ApplyRequest_Type = 204
 	ApplyRequest_TYPE_REPLICATION_REPLICATE_REMOVE                ApplyRequest_Type = 205
 	ApplyRequest_TYPE_REPLICATION_REPLICATE_CANCELLATION_COMPLETE ApplyRequest_Type = 206
-	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE_BY_COLLECTION  ApplyRequest_Type = 207
-	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE_BY_TENANTS     ApplyRequest_Type = 208
-	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE_ALL            ApplyRequest_Type = 209
+	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE_ALL            ApplyRequest_Type = 207
+	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE_BY_COLLECTION  ApplyRequest_Type = 208
+	ApplyRequest_TYPE_REPLICATION_REPLICATE_DELETE_BY_TENANTS     ApplyRequest_Type = 209
 	ApplyRequest_TYPE_REPLICATION_REPLICA_DISABLE                 ApplyRequest_Type = 210
 	ApplyRequest_TYPE_REPLICATION_REPLICA_DELETE                  ApplyRequest_Type = 211
 	ApplyRequest_TYPE_DISTRIBUTED_TASK_ADD                        ApplyRequest_Type = 300
@@ -101,9 +101,9 @@ var (
 		204: "TYPE_REPLICATION_REPLICATE_DELETE",
 		205: "TYPE_REPLICATION_REPLICATE_REMOVE",
 		206: "TYPE_REPLICATION_REPLICATE_CANCELLATION_COMPLETE",
-		207: "TYPE_REPLICATION_REPLICATE_DELETE_BY_COLLECTION",
-		208: "TYPE_REPLICATION_REPLICATE_DELETE_BY_TENANTS",
-		209: "TYPE_REPLICATION_REPLICATE_DELETE_ALL",
+		207: "TYPE_REPLICATION_REPLICATE_DELETE_ALL",
+		208: "TYPE_REPLICATION_REPLICATE_DELETE_BY_COLLECTION",
+		209: "TYPE_REPLICATION_REPLICATE_DELETE_BY_TENANTS",
 		210: "TYPE_REPLICATION_REPLICA_DISABLE",
 		211: "TYPE_REPLICATION_REPLICA_DELETE",
 		300: "TYPE_DISTRIBUTED_TASK_ADD",
@@ -143,9 +143,9 @@ var (
 		"TYPE_REPLICATION_REPLICATE_DELETE":                204,
 		"TYPE_REPLICATION_REPLICATE_REMOVE":                205,
 		"TYPE_REPLICATION_REPLICATE_CANCELLATION_COMPLETE": 206,
-		"TYPE_REPLICATION_REPLICATE_DELETE_BY_COLLECTION":  207,
-		"TYPE_REPLICATION_REPLICATE_DELETE_BY_TENANTS":     208,
-		"TYPE_REPLICATION_REPLICATE_DELETE_ALL":            209,
+		"TYPE_REPLICATION_REPLICATE_DELETE_ALL":            207,
+		"TYPE_REPLICATION_REPLICATE_DELETE_BY_COLLECTION":  208,
+		"TYPE_REPLICATION_REPLICATE_DELETE_BY_TENANTS":     209,
 		"TYPE_REPLICATION_REPLICA_DISABLE":                 210,
 		"TYPE_REPLICATION_REPLICA_DELETE":                  211,
 		"TYPE_DISTRIBUTED_TASK_ADD":                        300,
@@ -207,6 +207,7 @@ const (
 	QueryRequest_TYPE_GET_SHARDING_STATE_BY_COLLECTION                QueryRequest_Type = 204
 	QueryRequest_TYPE_GET_SHARDING_STATE_BY_COLLECTION_AND_SHARD      QueryRequest_Type = 205
 	QueryRequest_TYPE_GET_ALL_REPLICATION_DETAILS                     QueryRequest_Type = 206
+	QueryRequest_TYPE_GET_REPLICATION_OPERATION_STATE                 QueryRequest_Type = 207
 	QueryRequest_TYPE_DISTRIBUTED_TASK_LIST                           QueryRequest_Type = 300
 )
 
@@ -235,6 +236,7 @@ var (
 		204: "TYPE_GET_SHARDING_STATE_BY_COLLECTION",
 		205: "TYPE_GET_SHARDING_STATE_BY_COLLECTION_AND_SHARD",
 		206: "TYPE_GET_ALL_REPLICATION_DETAILS",
+		207: "TYPE_GET_REPLICATION_OPERATION_STATE",
 		300: "TYPE_DISTRIBUTED_TASK_LIST",
 	}
 	QueryRequest_Type_value = map[string]int32{
@@ -260,6 +262,7 @@ var (
 		"TYPE_GET_SHARDING_STATE_BY_COLLECTION":                204,
 		"TYPE_GET_SHARDING_STATE_BY_COLLECTION_AND_SHARD":      205,
 		"TYPE_GET_ALL_REPLICATION_DETAILS":                     206,
+		"TYPE_GET_REPLICATION_OPERATION_STATE":                 207,
 		"TYPE_DISTRIBUTED_TASK_LIST":                           300,
 	}
 )
@@ -1631,16 +1634,16 @@ var file_api_message_proto_rawDesc = []byte{
 	0x4f, 0x56, 0x45, 0x10, 0xcd, 0x01, 0x12, 0x35, 0x0a, 0x30, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52,
 	0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49,
 	0x43, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f,
-	0x4e, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0xce, 0x01, 0x12, 0x34, 0x0a,
-	0x2f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f,
+	0x4e, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0xce, 0x01, 0x12, 0x2a, 0x0a,
+	0x25, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f,
 	0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45,
-	0x54, 0x45, 0x5f, 0x42, 0x59, 0x5f, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e,
-	0x10, 0xcf, 0x01, 0x12, 0x31, 0x0a, 0x2c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4c,
-	0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54,
-	0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x59, 0x5f, 0x54, 0x45, 0x4e, 0x41,
-	0x4e, 0x54, 0x53, 0x10, 0xd0, 0x01, 0x12, 0x2a, 0x0a, 0x25, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52,
-	0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49,
-	0x43, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x41, 0x4c, 0x4c, 0x10,
+	0x54, 0x45, 0x5f, 0x41, 0x4c, 0x4c, 0x10, 0xcf, 0x01, 0x12, 0x34, 0x0a, 0x2f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45,
+	0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42,
+	0x59, 0x5f, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0xd0, 0x01, 0x12,
+	0x31, 0x0a, 0x2c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54,
+	0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x45,
+	0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x59, 0x5f, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x54, 0x53, 0x10,
 	0xd1, 0x01, 0x12, 0x25, 0x0a, 0x20, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49,
 	0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x5f, 0x44,
 	0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0xd2, 0x01, 0x12, 0x24, 0x0a, 0x1f, 0x54, 0x59, 0x50,
@@ -1659,7 +1662,7 @@ var file_api_message_proto_rawDesc = []byte{
 	0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65,
 	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72,
 	0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0xe9, 0x06, 0x0a,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x94, 0x07, 0x0a,
 	0x0c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a,
 	0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x77, 0x65,
 	0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e,
@@ -1667,7 +1670,7 @@ var file_api_message_proto_rawDesc = []byte{
 	0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
 	0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02,
 	0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64,
-	0x22, 0xf5, 0x05, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x59, 0x50,
+	0x22, 0xa0, 0x06, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x59, 0x50,
 	0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12,
 	0x14, 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x54, 0x5f, 0x43, 0x4c, 0x41, 0x53,
 	0x53, 0x45, 0x53, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45,
@@ -1712,153 +1715,156 @@ var file_api_message_proto_rawDesc = []byte{
 	0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x44, 0x10, 0xcd, 0x01,
 	0x12, 0x25, 0x0a, 0x20, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x54, 0x5f, 0x41, 0x4c, 0x4c,
 	0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x44, 0x45, 0x54,
-	0x41, 0x49, 0x4c, 0x53, 0x10, 0xce, 0x01, 0x12, 0x1f, 0x0a, 0x1a, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x44, 0x49, 0x53, 0x54, 0x52, 0x49, 0x42, 0x55, 0x54, 0x45, 0x44, 0x5f, 0x54, 0x41, 0x53, 0x4b,
-	0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0xac, 0x02, 0x22, 0x29, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72,
-	0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c,
-	0x6f, 0x61, 0x64, 0x22, 0x75, 0x0a, 0x11, 0x41, 0x64, 0x64, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74,
-	0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73,
-	0x74, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x3b, 0x0a,
-	0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21,
+	0x41, 0x49, 0x4c, 0x53, 0x10, 0xce, 0x01, 0x12, 0x29, 0x0a, 0x24, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x47, 0x45, 0x54, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f,
+	0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x10,
+	0xcf, 0x01, 0x12, 0x1f, 0x0a, 0x1a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x53, 0x54, 0x52,
+	0x49, 0x42, 0x55, 0x54, 0x45, 0x44, 0x5f, 0x54, 0x41, 0x53, 0x4b, 0x5f, 0x4c, 0x49, 0x53, 0x54,
+	0x10, 0xac, 0x02, 0x22, 0x29, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x75,
+	0x0a, 0x11, 0x41, 0x64, 0x64, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e,
+	0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73,
+	0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61,
+	0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x77, 0x65, 0x61, 0x76,
+	0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c,
+	0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x52, 0x07, 0x74, 0x65,
+	0x6e, 0x61, 0x6e, 0x74, 0x73, 0x22, 0x78, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54,
+	0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a,
+	0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21,
 	0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
 	0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e,
-	0x74, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x22, 0x78, 0x0a, 0x14, 0x55, 0x70,
-	0x64, 0x61, 0x74, 0x65, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20,
-	0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69,
-	0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
-	0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x12,
-	0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73,
-	0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e,
-	0x6f, 0x64, 0x65, 0x73, 0x22, 0xcc, 0x01, 0x0a, 0x0e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73,
-	0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3c, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69,
-	0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
-	0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, 0x4f,
-	0x70, 0x52, 0x02, 0x6f, 0x70, 0x12, 0x39, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65,
-	0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74,
-	0x22, 0x41, 0x0a, 0x02, 0x4f, 0x70, 0x12, 0x12, 0x0a, 0x0e, 0x4f, 0x50, 0x5f, 0x55, 0x4e, 0x53,
-	0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4f, 0x50,
-	0x5f, 0x53, 0x54, 0x41, 0x52, 0x54, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x50, 0x5f, 0x44,
-	0x4f, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x4f, 0x50, 0x5f, 0x41, 0x42, 0x4f, 0x52,
-	0x54, 0x10, 0x03, 0x22, 0xa0, 0x02, 0x0a, 0x14, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x50, 0x72,
-	0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04,
-	0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65,
-	0x12, 0x4e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e,
-	0x32, 0x36, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65,
+	0x74, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c,
+	0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x22,
+	0xcc, 0x01, 0x0a, 0x0e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x63, 0x65,
+	0x73, 0x73, 0x12, 0x3c, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c,
+	0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e,
+	0x74, 0x73, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x2e, 0x4f, 0x70, 0x52, 0x02, 0x6f, 0x70,
+	0x12, 0x39, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x21, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65,
 	0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e,
-	0x61, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x12, 0x56, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x63,
-	0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x77, 0x65,
+	0x61, 0x6e, 0x74, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x22, 0x41, 0x0a, 0x02, 0x4f,
+	0x70, 0x12, 0x12, 0x0a, 0x0e, 0x4f, 0x50, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46,
+	0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4f, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x52,
+	0x54, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x50, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02,
+	0x12, 0x0c, 0x0a, 0x08, 0x4f, 0x50, 0x5f, 0x41, 0x42, 0x4f, 0x52, 0x54, 0x10, 0x03, 0x22, 0xa0,
+	0x02, 0x0a, 0x14, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x61,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x77, 0x65,
 	0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e,
-	0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x50,
-	0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x50,
-	0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x4c, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x12, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53,
-	0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x41, 0x43,
-	0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x46, 0x52, 0x45, 0x45, 0x5a, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12,
-	0x15, 0x0a, 0x11, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x46, 0x52, 0x45, 0x45,
-	0x5a, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x22, 0x30, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
-	0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18,
-	0x0a, 0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x07, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x22, 0x34, 0x0a, 0x06, 0x54, 0x65, 0x6e, 0x61,
-	0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x9c,
-	0x01, 0x0a, 0x19, 0x41, 0x64, 0x64, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65,
-	0x64, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09,
-	0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79,
-	0x6c, 0x6f, 0x61, 0x64, 0x12, 0x37, 0x0a, 0x18, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65,
-	0x64, 0x5f, 0x61, 0x74, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73,
-	0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65,
-	0x64, 0x41, 0x74, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xe9, 0x01,
-	0x0a, 0x2a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75,
-	0x74, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c,
-	0x65, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09,
-	0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65,
-	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72,
-	0x73, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18,
-	0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a,
-	0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05,
-	0x65, 0x72, 0x72, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x17, 0x66, 0x69, 0x6e, 0x69,
-	0x73, 0x68, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x69, 0x6c,
-	0x6c, 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x66, 0x69, 0x6e, 0x69, 0x73,
-	0x68, 0x65, 0x64, 0x41, 0x74, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x42,
-	0x08, 0x0a, 0x06, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x9f, 0x01, 0x0a, 0x1c, 0x43, 0x61,
-	0x6e, 0x63, 0x65, 0x6c, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x54,
-	0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61,
-	0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e,
-	0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73,
-	0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
-	0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x18, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x5f,
+	0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x50, 0x72,
+	0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x41, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x56, 0x0a, 0x11, 0x74,
+	0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73,
+	0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74,
+	0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
+	0x65, 0x72, 0x2e, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73,
+	0x73, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73,
+	0x73, 0x65, 0x73, 0x22, 0x4c, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a,
+	0x12, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46,
+	0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f,
+	0x46, 0x52, 0x45, 0x45, 0x5a, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x41, 0x43,
+	0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x46, 0x52, 0x45, 0x45, 0x5a, 0x49, 0x4e, 0x47, 0x10,
+	0x02, 0x22, 0x30, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x65, 0x6e, 0x61, 0x6e,
+	0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x65, 0x6e,
+	0x61, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x6e, 0x61,
+	0x6e, 0x74, 0x73, 0x22, 0x34, 0x0a, 0x06, 0x54, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x12, 0x0a,
+	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x9c, 0x01, 0x0a, 0x19, 0x41, 0x64,
+	0x64, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73,
+	0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65,
+	0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12,
+	0x37, 0x0a, 0x18, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f,
+	0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x15, 0x73, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x41, 0x74, 0x55, 0x6e,
+	0x69, 0x78, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xe9, 0x01, 0x0a, 0x2a, 0x52, 0x65, 0x63,
+	0x6f, 0x72, 0x64, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x54, 0x61,
+	0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73,
+	0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65,
+	0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12,
+	0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f,
+	0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72,
+	0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x17, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f,
 	0x61, 0x74, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x06,
-	0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x41,
-	0x74, 0x55, 0x6e, 0x69, 0x78, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0x67, 0x0a, 0x1d, 0x43,
-	0x6c, 0x65, 0x61, 0x6e, 0x55, 0x70, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65,
-	0x64, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09,
-	0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65,
-	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72,
-	0x73, 0x69, 0x6f, 0x6e, 0x32, 0x8d, 0x04, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
-	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6b, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x76,
-	0x65, 0x50, 0x65, 0x65, 0x72, 0x12, 0x2c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65,
-	0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
-	0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69,
-	0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
-	0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x08, 0x4a, 0x6f, 0x69, 0x6e, 0x50, 0x65, 0x65, 0x72,
-	0x12, 0x2a, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65,
-	0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x4a, 0x6f, 0x69,
-	0x6e, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x77,
-	0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
-	0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x50, 0x65, 0x65,
-	0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6b, 0x0a, 0x0a, 0x4e,
-	0x6f, 0x74, 0x69, 0x66, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x2c, 0x2e, 0x77, 0x65, 0x61, 0x76,
-	0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c,
-	0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x50, 0x65, 0x65, 0x72,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x41, 0x74,
+	0x55, 0x6e, 0x69, 0x78, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x65,
+	0x72, 0x72, 0x6f, 0x72, 0x22, 0x9f, 0x01, 0x0a, 0x1c, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x44,
+	0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61,
+	0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70,
+	0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03,
+	0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a,
+	0x18, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x75, 0x6e,
+	0x69, 0x78, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x15, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x41, 0x74, 0x55, 0x6e, 0x69, 0x78,
+	0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0x67, 0x0a, 0x1d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x55,
+	0x70, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73,
+	0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65,
+	0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x32,
+	0x8d, 0x04, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x12, 0x6b, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x65, 0x65, 0x72,
+	0x12, 0x2c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65,
+	0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d,
+	0x6f, 0x76, 0x65, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d,
+	0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76,
+	0x65, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
+	0x65, 0x0a, 0x08, 0x4a, 0x6f, 0x69, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x12, 0x2a, 0x2e, 0x77, 0x65,
+	0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e,
+	0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x50, 0x65, 0x65, 0x72,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61,
 	0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73,
-	0x74, 0x65, 0x72, 0x2e, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c,
-	0x79, 0x12, 0x27, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74,
-	0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x70,
-	0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x77, 0x65, 0x61,
-	0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12,
-	0x27, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72,
-	0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72,
-	0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69,
-	0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75,
-	0x73, 0x74, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x73, 0x65, 0x22, 0x00, 0x42, 0xe1, 0x01, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x77, 0x65, 0x61,
+	0x74, 0x65, 0x72, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6b, 0x0a, 0x0a, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79,
+	0x50, 0x65, 0x65, 0x72, 0x12, 0x2c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e,
+	0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
+	0x2e, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e,
+	0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x4e,
+	0x6f, 0x74, 0x69, 0x66, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, 0x27, 0x2e, 0x77,
+	0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
+	0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65,
+	0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
+	0x72, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
+	0x00, 0x12, 0x5c, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x27, 0x2e, 0x77, 0x65, 0x61,
 	0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x42, 0x0c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50,
-	0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
-	0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76,
-	0x69, 0x61, 0x74, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x2f, 0x61, 0x70, 0x69, 0xa2, 0x02, 0x03, 0x57, 0x49, 0x43, 0xaa, 0x02, 0x19, 0x57, 0x65, 0x61,
-	0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43,
-	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0xca, 0x02, 0x19, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74,
-	0x65, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6c, 0x75, 0x73, 0x74,
-	0x65, 0x72, 0xe2, 0x02, 0x25, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x5c, 0x49, 0x6e,
-	0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5c, 0x47,
-	0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x1b, 0x57, 0x65, 0x61,
-	0x76, 0x69, 0x61, 0x74, 0x65, 0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3a,
-	0x3a, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x69,
+	0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
+	0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42,
+	0xe1, 0x01, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65,
+	0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
+	0x72, 0x42, 0x0c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50,
+	0x01, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x65,
+	0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f,
+	0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0xa2,
+	0x02, 0x03, 0x57, 0x49, 0x43, 0xaa, 0x02, 0x19, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65,
+	0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65,
+	0x72, 0xca, 0x02, 0x19, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x5c, 0x49, 0x6e, 0x74,
+	0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0xe2, 0x02, 0x25,
+	0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x5c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61,
+	0x6c, 0x5c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x1b, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65,
+	0x3a, 0x3a, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x3a, 0x3a, 0x43, 0x6c, 0x75, 0x73,
+	0x74, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/cluster/proto/api/message.proto b/cluster/proto/api/message.proto
index a2af4103d64..bbf9b69ad60 100644
--- a/cluster/proto/api/message.proto
+++ b/cluster/proto/api/message.proto
@@ -124,6 +124,7 @@ message QueryRequest {
     TYPE_GET_SHARDING_STATE_BY_COLLECTION = 204;
     TYPE_GET_SHARDING_STATE_BY_COLLECTION_AND_SHARD = 205;
     TYPE_GET_ALL_REPLICATION_DETAILS = 206;
+    TYPE_GET_REPLICATION_OPERATION_STATE = 207;
 
     TYPE_DISTRIBUTED_TASK_LIST = 300;
   }
diff --git a/cluster/proto/api/shard_requests.go b/cluster/proto/api/shard_requests.go
index cd9eaea0b72..eb0d409fab2 100644
--- a/cluster/proto/api/shard_requests.go
+++ b/cluster/proto/api/shard_requests.go
@@ -164,3 +164,11 @@ type ReplicationQueryShardingStateByCollectionAndShardRequest struct {
 type ReplicationDeleteAllRequest struct {
 	Version int
 }
+
+type ReplicationOperationStateRequest struct {
+	Id uint64
+}
+
+type ReplicationOperationStateResponse struct {
+	State ShardReplicationState
+}
diff --git a/cluster/raft_replication_query_endpoints.go b/cluster/raft_replication_query_endpoints.go
index e29d0e0e67c..2700da83936 100644
--- a/cluster/raft_replication_query_endpoints.go
+++ b/cluster/raft_replication_query_endpoints.go
@@ -238,3 +238,35 @@ func (s *Raft) QueryShardingStateByCollectionAndShard(collection string, shard s
 
 	return response, nil
 }
+
+func (s *Raft) ReplicationGetReplicaOpStatus(id uint64) (api.ShardReplicationState, error) {
+	request := &api.ReplicationOperationStateRequest{
+		Id: id,
+	}
+
+	subCommand, err := json.Marshal(request)
+	if err != nil {
+		return "", fmt.Errorf("marshal request: %w", err)
+	}
+
+	command := &api.QueryRequest{
+		Type:       api.QueryRequest_TYPE_GET_REPLICATION_OPERATION_STATE,
+		SubCommand: subCommand,
+	}
+
+	queryResponse, err := s.Query(context.Background(), command)
+	if err != nil {
+		if strings.Contains(err.Error(), replicationTypes.ErrReplicationOperationNotFound.Error()) {
+			return "", fmt.Errorf("%w: %w", types.ErrNotFound, replicationTypes.ErrReplicationOperationNotFound)
+		}
+		return "", fmt.Errorf("failed to execute query: %w", err)
+	}
+
+	response := api.ReplicationOperationStateResponse{}
+	err = json.Unmarshal(queryResponse.Payload, &response)
+	if err != nil {
+		return "", fmt.Errorf("failed to unmarshal query response: %w", err)
+	}
+
+	return response.State, nil
+}
diff --git a/cluster/replication/consumer.go b/cluster/replication/consumer.go
index f0c4dabad1e..9c21d753e7c 100644
--- a/cluster/replication/consumer.go
+++ b/cluster/replication/consumer.go
@@ -179,13 +179,34 @@ func (c *CopyOpConsumer) Consume(ctx context.Context, in <-chan ShardReplication
 			// running replication operations and avoids overloading the system.
 			case c.tokens <- struct{}{}:
 
-				// Check if the operation is already in progress
+				shouldSkip := false
 				if c.ongoingOps.LoadOrStore(op.Op.ID) {
+					// Check if the operation is already in progress
 					// Avoid scheduling unnecessary work or incorrectly counting metrics
 					// for operations that are already in progress or completed.
 					c.logger.WithFields(logrus.Fields{"op": op}).Debug("replication op skipped as already running")
-					// Need to release the token to let other consumers process queued replication operations.
+					shouldSkip = true
+				} else {
+					// Check if the operation has had its state changed between being added to the channel and being processed
+					// This is chatty and will likely cause a lot of unnecessary load on the leader
+					// For now, we need it to ensure eventual consistency between the FSM and the consumer
+					state, err := c.leaderClient.ReplicationGetReplicaOpStatus(op.Op.ID)
+					if err != nil {
+						c.logger.WithFields(logrus.Fields{"op": op}).Error("error while checking status of replication op")
+						shouldSkip = true
+					} else if state.String() != op.Status.GetCurrent().State.String() {
+						c.logger.WithFields(logrus.Fields{"op": op}).Debug("replication op skipped as state has changed")
+						shouldSkip = true
+					}
+				}
+				// TODO: Consider more optimal ways of checking that the state of the op has not changed between it being added to the channel
+				// and being processed here. Could use in-memory solution, e.g. using cache, or refactor consumer-producer to be event/notification-based
+				// For now, ensure consistency by checking the FSM through the leader
+
+				// Need to release the token to let other consumers process queued replication operations.
+				if shouldSkip {
 					<-c.tokens
+					c.ongoingOps.DeleteInFlight(op.Op.ID)
 					c.engineOpCallbacks.OnOpSkipped(c.nodeId)
 					continue
 				}
diff --git a/cluster/replication/consumer_test.go b/cluster/replication/consumer_test.go
index 904db4c3552..e6f6485c49e 100644
--- a/cluster/replication/consumer_test.go
+++ b/cluster/replication/consumer_test.go
@@ -42,6 +42,10 @@ func TestConsumerWithCallbacks(t *testing.T) {
 		opId, err := randInt(t, 100, 200)
 		require.NoError(t, err, "error generating random operation id")
 
+		mockFSMUpdater.EXPECT().
+			ReplicationGetReplicaOpStatus(uint64(opId)).
+			Return(api.REGISTERED, nil).
+			Times(1)
 		mockFSMUpdater.EXPECT().
 			ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 			Return(nil)
@@ -180,6 +184,10 @@ func TestConsumerWithCallbacks(t *testing.T) {
 		opId, err := randInt(t, 100, 200)
 		require.NoError(t, err, "error generating random operation id")
 
+		mockFSMUpdater.EXPECT().
+			ReplicationGetReplicaOpStatus(uint64(opId)).
+			Return(api.REGISTERED, nil).
+			Times(1)
 		mockFSMUpdater.EXPECT().
 			ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 			Return(nil)
@@ -300,6 +308,10 @@ func TestConsumerWithCallbacks(t *testing.T) {
 		require.NoError(t, err, "error while generating random op id start")
 		for i := 0; i < randomNumberOfOps; i++ {
 			opId := uint64(randomStartOpId + i)
+			mockFSMUpdater.EXPECT().
+				ReplicationGetReplicaOpStatus(uint64(opId)).
+				Return(api.REGISTERED, nil).
+				Times(1)
 			mockFSMUpdater.EXPECT().
 				ReplicationUpdateReplicaOpStatus(opId, api.HYDRATING).
 				Return(nil)
@@ -574,6 +586,10 @@ func TestConsumerWithCallbacks(t *testing.T) {
 			if !skip {
 				expectedStarted++
 				expectedCompleted++
+				mockFSMUpdater.EXPECT().
+					ReplicationGetReplicaOpStatus(uint64(opID)).
+					Return(api.REGISTERED, nil).
+					Times(1)
 				mockFSMUpdater.EXPECT().
 					ReplicationUpdateReplicaOpStatus(opID, api.HYDRATING).
 					Return(nil)
@@ -787,7 +803,12 @@ func TestConsumerOpCancellation(t *testing.T) {
 		}).Maybe()
 
 	op := replication.NewShardReplicationOp(1, "node1", "node2", "TestCollection", "test-shard", api.COPY)
+
 	status := replication.NewShardReplicationStatus(api.HYDRATING)
+	mockFSMUpdater.EXPECT().
+		ReplicationGetReplicaOpStatus(uint64(1)).
+		Return(api.HYDRATING, nil)
+
 	status.TriggerCancellation()
 	// Simulate the copying step that will loop forever until cancelled in the mock
 	opsChan <- replication.NewShardReplicationOpAndStatus(op, status)
@@ -899,7 +920,12 @@ func TestConsumerOpDeletion(t *testing.T) {
 		}).Maybe()
 
 	op := replication.NewShardReplicationOp(1, "node1", "node2", "TestCollection", "test-shard", api.COPY)
+
 	status := replication.NewShardReplicationStatus(api.HYDRATING)
+	mockFSMUpdater.EXPECT().
+		ReplicationGetReplicaOpStatus(uint64(1)).
+		Return(api.HYDRATING, nil)
+
 	status.TriggerDeletion()
 	// Simulate the copying step that will loop forever until cancelled in the mock
 	opsChan <- replication.NewShardReplicationOpAndStatus(op, status)
@@ -929,3 +955,142 @@ func TestConsumerOpDeletion(t *testing.T) {
 	mockFSMUpdater.AssertExpectations(t)
 	mockReplicaCopier.AssertExpectations(t)
 }
+
+func TestConsumerOpDuplication(t *testing.T) {
+	// GIVEN
+	logger, _ := logrustest.NewNullLogger()
+	mockFSMUpdater := types.NewMockFSMUpdater(t)
+	mockReplicaCopier := types.NewMockReplicaCopier(t)
+
+	var completionWg sync.WaitGroup
+	metricsCallbacks := metrics.NewReplicationEngineOpsCallbacksBuilder().
+		WithPrepareProcessing(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in prepare processing callback")
+		}).
+		WithOpPendingCallback(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in pending op callback")
+		}).
+		WithOpSkippedCallback(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in skipped op callback")
+			completionWg.Done()
+		}).
+		WithOpStartCallback(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in start op callback")
+		}).
+		WithOpCompleteCallback(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in complete op callback")
+			completionWg.Done()
+		}).
+		WithOpFailedCallback(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in failed op callback")
+		}).
+		WithOpCancelledCallback(func(node string) {
+			require.Equal(t, "node2", node, "invalid node in cancelled op callback")
+		}).
+		Build()
+
+	consumer := replication.NewCopyOpConsumer(
+		logger,
+		mockFSMUpdater,
+		mockReplicaCopier,
+		"node2",
+		&backoff.StopBackOff{},
+		replication.NewOpsCache(),
+		time.Second*10,
+		1,
+		runtime.NewDynamicValue(time.Second*100),
+		metricsCallbacks,
+	)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	opsChan := make(chan replication.ShardReplicationOpAndStatus, 2)
+	doneChan := make(chan error, 1)
+
+	// WHEN
+	go func() {
+		doneChan <- consumer.Consume(ctx, opsChan)
+	}()
+
+	mockReplicaCopier.EXPECT().
+		CopyReplica(mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+		RunAndReturn(func(ctx context.Context, sourceNode string, collectionName string, shardName string) error {
+			// Simulate a long-running operation that checks for cancellation every loop
+			start := time.Now()
+			for {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				if time.Since(start) > 10*time.Second {
+					break
+				}
+			}
+			return nil
+		}).Times(1)
+	mockFSMUpdater.EXPECT().
+		ReplicationGetReplicaOpStatus(uint64(1)).
+		Return(api.HYDRATING, nil).
+		Times(1)
+	mockFSMUpdater.EXPECT().
+		ReplicationGetReplicaOpStatus(uint64(1)).
+		Return(api.FINALIZING, nil).
+		Times(1)
+	mockFSMUpdater.EXPECT().
+		ReplicationUpdateReplicaOpStatus(uint64(1), api.FINALIZING).
+		Return(nil).
+		Times(1)
+	mockFSMUpdater.EXPECT().
+		ReplicationUpdateReplicaOpStatus(uint64(1), api.READY).
+		Return(nil).
+		Times(1)
+	mockFSMUpdater.EXPECT().
+		AddReplicaToShard(mock.Anything, "TestCollection", mock.Anything, "node2").
+		Return(uint64(1), nil)
+	mockReplicaCopier.EXPECT().
+		AsyncReplicationStatus(mock.Anything, "node1", "node2", "TestCollection", mock.Anything).
+		Return(models.AsyncReplicationStatus{
+			ObjectsPropagated:       0,
+			StartDiffTimeUnixMillis: time.Now().Add(200 * time.Second).UnixMilli(),
+		}, nil)
+	mockReplicaCopier.EXPECT().
+		AddAsyncReplicationTargetNode(mock.Anything, mock.Anything).Return(nil)
+	mockReplicaCopier.EXPECT().
+		RemoveAsyncReplicationTargetNode(mock.Anything, mock.Anything).Return(nil)
+	mockReplicaCopier.EXPECT().
+		InitAsyncReplicationLocally(mock.Anything, mock.Anything, mock.Anything).
+		Return(nil)
+	mockReplicaCopier.EXPECT().
+		RevertAsyncReplicationLocally(mock.Anything, mock.Anything, mock.Anything).Return(nil)
+
+	op := replication.NewShardReplicationOp(1, "node1", "node2", "TestCollection", "test-shard", api.COPY)
+	status := replication.NewShardReplicationStatus(api.HYDRATING)
+
+	// Simulate the copying step that will loop for 10s before exiting
+	opsChan <- replication.NewShardReplicationOpAndStatus(op, status)
+	completionWg.Add(1)
+	// Send the same operation again to make sure it isn't reprocessed
+	opsChan <- replication.NewShardReplicationOpAndStatus(op, status)
+	completionWg.Add(1)
+
+	waitChan := make(chan struct{})
+	go func() {
+		completionWg.Wait()
+		waitChan <- struct{}{}
+	}()
+
+	select {
+	case <-waitChan:
+	case <-time.After(30 * time.Second):
+		t.Fatalf("Test timed out waiting for operation completion")
+	}
+
+	close(opsChan)
+	err := <-doneChan
+
+	// THEN
+	require.NoError(t, err, "expected consumer to stop without error")
+
+	mockFSMUpdater.AssertExpectations(t)
+	mockReplicaCopier.AssertExpectations(t)
+}
diff --git a/cluster/replication/manager.go b/cluster/replication/manager.go
index b5e49e553a1..a2dbf53fe6f 100644
--- a/cluster/replication/manager.go
+++ b/cluster/replication/manager.go
@@ -72,7 +72,15 @@ func (m *Manager) RegisterError(c *cmd.ApplyRequest) error {
 	}
 
 	// Store an op's error emitted by the consumer in the FSM
-	return m.replicationFSM.RegisterError(req)
+	if err := m.replicationFSM.RegisterError(req); err != nil {
+		if errors.Is(err, ErrMaxErrorsReached) {
+			return m.replicationFSM.CancelReplication(&cmd.ReplicationCancelRequest{
+				Uuid: req.Uuid,
+			})
+		}
+		return err
+	}
+	return nil
 }
 
 func (m *Manager) UpdateReplicateOpState(c *cmd.ApplyRequest) error {
@@ -109,6 +117,28 @@ func (m *Manager) GetReplicationDetailsByReplicationId(c *cmd.QueryRequest) ([]b
 	return payload, nil
 }
 
+func (m *Manager) GetReplicationOperationState(c *cmd.QueryRequest) ([]byte, error) {
+	subCommand := cmd.ReplicationOperationStateRequest{}
+	if err := json.Unmarshal(c.SubCommand, &subCommand); err != nil {
+		return nil, fmt.Errorf("%w: %w", ErrBadRequest, err)
+	}
+
+	status, ok := m.replicationFSM.statusById[subCommand.Id]
+	if !ok {
+		return nil, fmt.Errorf("unable to retrieve replication operation '%d' status", subCommand.Id)
+	}
+
+	response := cmd.ReplicationOperationStateResponse{
+		State: status.GetCurrent().State,
+	}
+	payload, err := json.Marshal(response)
+	if err != nil {
+		return nil, fmt.Errorf("could not marshal query response for replication operation '%d': %w", subCommand.Id, err)
+	}
+
+	return payload, nil
+}
+
 func (m *Manager) GetReplicationDetailsByCollection(c *cmd.QueryRequest) ([]byte, error) {
 	subCommand := cmd.ReplicationDetailsRequestByCollection{}
 	if err := json.Unmarshal(c.SubCommand, &subCommand); err != nil {
diff --git a/cluster/replication/producer.go b/cluster/replication/producer.go
index c00c0bc702f..d4cfd12b661 100644
--- a/cluster/replication/producer.go
+++ b/cluster/replication/producer.go
@@ -84,10 +84,18 @@ func (p *FSMOpProducer) Produce(ctx context.Context, out chan<- ShardReplication
 			p.logger.WithFields(logrus.Fields{"number_of_ops": len(ops)}).Debug("preparing op replication")
 
 			for _, op := range ops {
+				status, ok := p.fsm.GetOpState(op) // Get most recent state to narrow the window for state change races between producer and consumer
+				if !ok {
+					p.logger.WithField("op", op).Debug("skipping op as it has no state stored in FSM. It may have been deleted in the meantime.")
+					continue
+				}
+				if !status.ShouldConsumeOps() {
+					continue
+				}
 				select {
 				case <-ctx.Done():
 					return ctx.Err()
-				case out <- op: // Write replication operation to channel.
+				case out <- NewShardReplicationOpAndStatus(op, status): // Write replication operation to channel.
 				}
 			}
 		}
@@ -108,13 +116,13 @@ func (p *FSMOpProducer) Produce(ctx context.Context, out chan<- ShardReplication
 //   - All states except for ABORTED and READY are processes
 //
 // Returns only operations that should be actively processed by this node.
-func (p *FSMOpProducer) allOpsForNode(nodeId string) []ShardReplicationOpAndStatus {
+func (p *FSMOpProducer) allOpsForNode(nodeId string) []ShardReplicationOp {
 	allNodeAsTargetOps := p.fsm.GetOpsForTarget(nodeId)
 
-	nodeOpsSubset := make([]ShardReplicationOpAndStatus, 0, len(allNodeAsTargetOps))
+	nodeOpsSubset := make([]ShardReplicationOp, 0, len(allNodeAsTargetOps))
 	for _, op := range allNodeAsTargetOps {
 		if opState, ok := p.fsm.GetOpState(op); ok && opState.ShouldConsumeOps() {
-			nodeOpsSubset = append(nodeOpsSubset, NewShardReplicationOpAndStatus(op, opState))
+			nodeOpsSubset = append(nodeOpsSubset, op)
 		} else if !ok {
 			p.logger.WithField("op", op).Warn("skipping op as it has no state stored in FSM. It may have been deleted in the meantime.")
 		}
diff --git a/cluster/replication/producer_consumer_test.go b/cluster/replication/producer_consumer_test.go
index 50dee4e1e0b..03f17e2e488 100644
--- a/cluster/replication/producer_consumer_test.go
+++ b/cluster/replication/producer_consumer_test.go
@@ -53,6 +53,10 @@ func TestConsumerStateChangeOrder(t *testing.T) {
 			transferType: api.COPY,
 			setupMocksFunc: func(wg *sync.WaitGroup, mockFSMUpdater *types.MockFSMUpdater, mockReplicaCopier *types.MockReplicaCopier) {
 				wg.Add(1)
+				mockFSMUpdater.EXPECT().
+					ReplicationGetReplicaOpStatus(uint64(opId)).
+					Return(api.REGISTERED, nil).
+					Times(1)
 				mockFSMUpdater.EXPECT().
 					ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 					Return(nil)
@@ -94,6 +98,10 @@ func TestConsumerStateChangeOrder(t *testing.T) {
 			transferType: api.COPY,
 			setupMocksFunc: func(wg *sync.WaitGroup, mockFSMUpdater *types.MockFSMUpdater, mockReplicaCopier *types.MockReplicaCopier) {
 				wg.Add(1)
+				mockFSMUpdater.EXPECT().
+					ReplicationGetReplicaOpStatus(uint64(opId)).
+					Return(api.REGISTERED, nil).
+					Times(2) // equal to the op plus number of times the op failed
 				mockFSMUpdater.EXPECT().
 					ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 					Return(fmt.Errorf("failed to update state")).
@@ -139,6 +147,10 @@ func TestConsumerStateChangeOrder(t *testing.T) {
 			transferType: api.COPY,
 			setupMocksFunc: func(wg *sync.WaitGroup, mockFSMUpdater *types.MockFSMUpdater, mockReplicaCopier *types.MockReplicaCopier) {
 				wg.Add(1)
+				mockFSMUpdater.EXPECT().
+					ReplicationGetReplicaOpStatus(uint64(opId)).
+					Return(api.REGISTERED, nil).
+					Times(2) // equal to the op plus number of times the op failed
 				mockFSMUpdater.EXPECT().
 					ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 					Return(nil)
@@ -187,6 +199,10 @@ func TestConsumerStateChangeOrder(t *testing.T) {
 			transferType: api.COPY,
 			setupMocksFunc: func(wg *sync.WaitGroup, mockFSMUpdater *types.MockFSMUpdater, mockReplicaCopier *types.MockReplicaCopier) {
 				wg.Add(1)
+				mockFSMUpdater.EXPECT().
+					ReplicationGetReplicaOpStatus(uint64(opId)).
+					Return(api.REGISTERED, nil).
+					Times(4) // equal to the op plus number of times the op failed
 				mockFSMUpdater.EXPECT().
 					ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 					Return(nil)
@@ -247,6 +263,10 @@ func TestConsumerStateChangeOrder(t *testing.T) {
 			transferType: api.MOVE,
 			setupMocksFunc: func(wg *sync.WaitGroup, mockFSMUpdater *types.MockFSMUpdater, mockReplicaCopier *types.MockReplicaCopier) {
 				wg.Add(1)
+				mockFSMUpdater.EXPECT().
+					ReplicationGetReplicaOpStatus(uint64(opId)).
+					Return(api.REGISTERED, nil).
+					Times(1)
 				mockFSMUpdater.EXPECT().
 					ReplicationUpdateReplicaOpStatus(uint64(opId), api.HYDRATING).
 					Return(nil)
diff --git a/cluster/replication/types/mock_fsm_updater.go b/cluster/replication/types/mock_fsm_updater.go
index c401ff2eb98..a0b100ed594 100644
--- a/cluster/replication/types/mock_fsm_updater.go
+++ b/cluster/replication/types/mock_fsm_updater.go
@@ -198,6 +198,62 @@ func (_c *MockFSMUpdater_ReplicationCancellationComplete_Call) RunAndReturn(run
 	return _c
 }
 
+// ReplicationGetReplicaOpStatus provides a mock function with given fields: id
+func (_m *MockFSMUpdater) ReplicationGetReplicaOpStatus(id uint64) (api.ShardReplicationState, error) {
+	ret := _m.Called(id)
+
+	if len(ret) == 0 {
+		panic("no return value specified for ReplicationGetReplicaOpStatus")
+	}
+
+	var r0 api.ShardReplicationState
+	var r1 error
+	if rf, ok := ret.Get(0).(func(uint64) (api.ShardReplicationState, error)); ok {
+		return rf(id)
+	}
+	if rf, ok := ret.Get(0).(func(uint64) api.ShardReplicationState); ok {
+		r0 = rf(id)
+	} else {
+		r0 = ret.Get(0).(api.ShardReplicationState)
+	}
+
+	if rf, ok := ret.Get(1).(func(uint64) error); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// MockFSMUpdater_ReplicationGetReplicaOpStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReplicationGetReplicaOpStatus'
+type MockFSMUpdater_ReplicationGetReplicaOpStatus_Call struct {
+	*mock.Call
+}
+
+// ReplicationGetReplicaOpStatus is a helper method to define mock.On call
+//   - id uint64
+func (_e *MockFSMUpdater_Expecter) ReplicationGetReplicaOpStatus(id interface{}) *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call {
+	return &MockFSMUpdater_ReplicationGetReplicaOpStatus_Call{Call: _e.mock.On("ReplicationGetReplicaOpStatus", id)}
+}
+
+func (_c *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call) Run(run func(id uint64)) *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call {
+	_c.Call.Run(func(args mock.Arguments) {
+		run(args[0].(uint64))
+	})
+	return _c
+}
+
+func (_c *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call) Return(_a0 api.ShardReplicationState, _a1 error) *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call {
+	_c.Call.Return(_a0, _a1)
+	return _c
+}
+
+func (_c *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call) RunAndReturn(run func(uint64) (api.ShardReplicationState, error)) *MockFSMUpdater_ReplicationGetReplicaOpStatus_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
 // ReplicationRegisterError provides a mock function with given fields: id, errorToRegister
 func (_m *MockFSMUpdater) ReplicationRegisterError(id uint64, errorToRegister string) error {
 	ret := _m.Called(id, errorToRegister)
diff --git a/cluster/replication/types/replication_fsm_updater.go b/cluster/replication/types/replication_fsm_updater.go
index baa135f2c44..0407a1f42b1 100644
--- a/cluster/replication/types/replication_fsm_updater.go
+++ b/cluster/replication/types/replication_fsm_updater.go
@@ -24,4 +24,5 @@ type FSMUpdater interface {
 	ReplicationRegisterError(id uint64, errorToRegister string) error
 	ReplicationRemoveReplicaOp(id uint64) error
 	ReplicationCancellationComplete(id uint64) error
+	ReplicationGetReplicaOpStatus(id uint64) (api.ShardReplicationState, error)
 }
diff --git a/cluster/service.go b/cluster/service.go
index 2f61b9e12de..20dfdc63b90 100644
--- a/cluster/service.go
+++ b/cluster/service.go
@@ -38,7 +38,7 @@ const (
 	// TODO: consider exposing these as settings
 	shardReplicationEngineBufferSize = 16
 	fsmOpProducerPollingInterval     = 5 * time.Second
-	replicationEngineShutdownTimeout = 10 * time.Minute
+	replicationEngineShutdownTimeout = 20 * time.Second
 	replicationOperationTimeout      = 24 * time.Hour
 	catchUpInterval                  = 5 * time.Second
 )
@@ -57,10 +57,9 @@ type Service struct {
 	logger    *logrus.Logger
 
 	// closing channels
-	closeBootstrapper       chan struct{}
-	closeOnFSMCaughtUp      chan struct{}
-	closeWaitForDB          chan struct{}
-	replicationEngineCancel context.CancelFunc
+	closeBootstrapper  chan struct{}
+	closeOnFSMCaughtUp chan struct{}
+	closeWaitForDB     chan struct{}
 }
 
 // New returns a Service configured with cfg. The service will initialize internals gRPC api & clients to other cluster
@@ -127,10 +126,9 @@ func (c *Service) onFSMCaughtUp(ctx context.Context) {
 		case <-ticker.C:
 			if c.Raft.store.FSMHasCaughtUp() {
 				c.logger.Infof("Metadata FSM reported caught up, starting replication engine")
-				replicationEngineCtx, replicationEngineCancel := context.WithCancel(ctx)
-				c.replicationEngineCancel = replicationEngineCancel
 				enterrors.GoWrapper(func() {
-					if err := c.replicationEngine.Start(replicationEngineCtx); err != nil {
+					// The context is cancelled by the engine itself when it is stopped
+					if err := c.replicationEngine.Start(context.Background()); err != nil {
 						c.logger.WithError(err).Error("replication engine failed to start after FSM caught up")
 					}
 				}, c.logger)
@@ -211,10 +209,8 @@ func (c *Service) Close(ctx context.Context) error {
 		c.closeOnFSMCaughtUp <- struct{}{}
 	}, c.logger)
 
-	if c.replicationEngineCancel != nil {
-		c.logger.Infof("Closing replication engine %v", c)
-		c.replicationEngineCancel()
-	}
+	c.logger.Info("closing replication engine ...")
+	c.replicationEngine.Stop()
 
 	c.logger.Info("closing raft FSM store ...")
 	if err := c.Raft.Close(ctx); err != nil {
diff --git a/cluster/store_query.go b/cluster/store_query.go
index 5674da8170a..92c33f77742 100644
--- a/cluster/store_query.go
+++ b/cluster/store_query.go
@@ -136,6 +136,11 @@ func (st *Store) Query(req *cmd.QueryRequest) (*cmd.QueryResponse, error) {
 		if err != nil {
 			return &cmd.QueryResponse{}, fmt.Errorf("could not get distributed task list: %w", err)
 		}
+	case cmd.QueryRequest_TYPE_GET_REPLICATION_OPERATION_STATE:
+		payload, err = st.replicationManager.GetReplicationOperationState(req)
+		if err != nil {
+			return &cmd.QueryResponse{}, fmt.Errorf("could not get replication operation state: %w", err)
+		}
 	default:
 		// This could occur when a new command has been introduced in a later app version
 		// At this point, we need to panic so that the app undergo an upgrade during restart


Reviews:
user: login: orca-security-eu[bot]
body: ### Orca Security Scan Summary
| Status  | Check | Issues by priority |   |
| ------- | ----- | ------------------ | - |
| <img width="16" alt="Passed" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/status_v1/passed.png" title="Passed"> Passed | Secrets | <img width="12" alt="high" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/high.png" title="High"> 0 &emsp; <img width="12" alt="medium" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/medium.png" title="Medium"> 0 &emsp; <img width="12" alt="low" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/low.png" title="Low"> 0 &emsp; <img width="12" alt="info" src="https://raw.githubusercontent.com/orcasecurity/orca-cli/main/resources/images/prcomment/priority/info.png" title="Info"> 0 | <a href="https://eu.app.orcasecurity.io/shift-left/file_system_secret_detection/scan-log/0a800b80-6c75-490e-9b8f-558da0acff32" target="_blank">View in Orca</a> |
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T15:11:31Z
user: login: reyreaud-l
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T16:30:10Z
user: login: reyreaud-l
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T16:35:38Z
user: login: reyreaud-l
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-18T09:28:28Z

Review comments:
diff_hunk: @@ -179,13 +179,29 @@ func (c *CopyOpConsumer) Consume(ctx context.Context, in <-chan ShardReplication
 			// running replication operations and avoids overloading the system.
 			case c.tokens <- struct{}{}:
 
-				// Check if the operation is already in progress
+				shouldSkip := false
 				if c.ongoingOps.LoadOrStore(op.Op.ID) {
+					// Check if the operation is already in progress
 					// Avoid scheduling unnecessary work or incorrectly counting metrics
 					// for operations that are already in progress or completed.
 					c.logger.WithFields(logrus.Fields{"op": op}).Debug("replication op skipped as already running")
-					// Need to release the token to let other consumers process queued replication operations.
+					shouldSkip = true
+				} else if state, err := c.leaderClient.ReplicationGetReplicaOpStatus(op.Op.ID); err != nil {
+					// Check if the operation has had its state changed between being added to the channel and being processed
+					c.logger.WithFields(logrus.Fields{"op": op}).Error("error while checking status of replication op")
+					shouldSkip = true
+				} else if state != op.Status.GetCurrent().State {
path: cluster/replication/consumer.go
user: login: reyreaud-l
body: Not sure i'm a huge fan of the subsequent else if initiating then reading values, but it gets the job done. 
Maybe

```
if c.Ongoing... {

} else {
state = call to leader
if state != currentState {
abort
}
}
```
created_at: 2025-05-16T16:30:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -66,8 +66,9 @@ func TestReplicationTestSuite(t *testing.T) {
 	suite.Run(t, new(ReplicationTestSuite))
 }
 
-func (suite *ReplicationTestSuite) TestReplicaMovementHappyPath() {
-	t := suite.T()
+//	func (suite *ReplicationTestSuite) TestReplicaMovementHappyPath() {
+//		t := suite.T()
+func TestReplicaMovementHappyPath(t *testing.T) {
path: test/acceptance/replication/replica_replication/replica_replication_test.go
user: login: reyreaud-l
body: Debugging leftover
created_at: 2025-05-16T16:35:38Z
author_association: CONTRIBUTOR

PR comments:
user: login: weaviate-git-bot
created_at: 2025-05-17T10:00:11Z
body: Great to see you again! Thanks for the contribution.<br><br>beep boop - the Weaviate bot 👋🤖<br><br>PS:<br>Are you already a member of the <a target="_blank" href="https://weaviate.io/slack">Weaviate Slack channel?</a>
author_association: NONE
user: login: sonarqubecloud[bot]
created_at: 2025-05-18T08:53:05Z
body: ## [![Quality Gate Failed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-failed-20px.png 'Quality Gate Failed')](https://sonarcloud.io/dashboard?id=weaviate_weaviate&pullRequest=8173) **Quality Gate failed**  
Failed conditions  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/failed-16px.png '') [15.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=weaviate_weaviate&pullRequest=8173&metric=new_duplicated_lines_density&view=list) (required ≤ 3%)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=weaviate_weaviate&pullRequest=8173)


author_association: NONE