Repository: radius-project/radius

Issue:
Document ARM RPC sync/async operation API flow
## Summary

[ARM RPC](https://github.com/Azure/azure-resource-manager-rpc/tree/master/v1.0) doesn't describe the async/sync operation API flow clearly. In particular, async operation spec misses API flow between client and RP. This issue is to write the radius development document which describes the API flow for sync/async API clearly.

- [ ] Document Sync/Async operation API flow
- [ ] Document Async Operation timeout / retry policy


[AB#5393](https://dev.azure.com/azure-octo/e61041b4-555f-47ae-95b2-4f8ab480ea57/_workitems/edit/5393)

Hints from issue comments:
@nicolejms: very old so closing

Solution patch:
diff --git a/pkg/armrpc/asyncoperation/worker/worker.go b/pkg/armrpc/asyncoperation/worker/worker.go
index 5079a3a08..ad85a254b 100644
--- a/pkg/armrpc/asyncoperation/worker/worker.go
+++ b/pkg/armrpc/asyncoperation/worker/worker.go
@@ -261,7 +261,10 @@ func (w *AsyncRequestProcessWorker) runOperation(ctx context.Context, message *q
 			logger.Info("Cancelling async operation.")
 
 			opCancel()
-			w.completeOperation(ctx, message, ctrl.NewCanceledResult("async operation timeout"), asyncCtrl.StorageClient())
+			errMessage := fmt.Sprintf("Operation (%s) has timed out because it was processing longer than %d s.", asyncReq.OperationType, int(asyncReq.Timeout().Seconds()))
+			result := ctrl.NewCanceledResult(errMessage)
+			result.Error.Target = asyncReq.ResourceID
+			w.completeOperation(ctx, message, result, asyncCtrl.StorageClient())
 			return
 
 		case <-ctx.Done():
diff --git a/pkg/corerp/frontend/controller/containers/createorupdatecontainer.go b/pkg/corerp/frontend/controller/containers/createorupdatecontainer.go
index 15dbc11c3..42ca94849 100644
--- a/pkg/corerp/frontend/controller/containers/createorupdatecontainer.go
+++ b/pkg/corerp/frontend/controller/containers/createorupdatecontainer.go
@@ -25,7 +25,7 @@ var _ ctrl.Controller = (*CreateOrUpdateContainer)(nil)
 
 var (
 	// AsyncPutContainerOperationTimeout is the default timeout duration of async put container operation.
-	AsyncPutContainerOperationTimeout = time.Duration(120) * time.Second
+	AsyncPutContainerOperationTimeout = time.Duration(5) * time.Minute
 )
 
 // CreateOrUpdateContainer is the controller implementation to create or update a container resource.



Test patch:
diff --git a/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go b/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go
index c6e82b740..36c67ba96 100644
--- a/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go
+++ b/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go
@@ -7,7 +7,9 @@ package worker
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -17,9 +19,11 @@ import (
 	ctrl "github.com/project-radius/radius/pkg/armrpc/asyncoperation/controller"
 	manager "github.com/project-radius/radius/pkg/armrpc/asyncoperation/statusmanager"
 	"github.com/project-radius/radius/pkg/corerp/backend/deployment"
+	"github.com/project-radius/radius/pkg/radrp/armerrors"
 	"github.com/project-radius/radius/pkg/ucp/dataprovider"
 	queue "github.com/project-radius/radius/pkg/ucp/queue/client"
 	"github.com/project-radius/radius/pkg/ucp/queue/inmemory"
+	"github.com/project-radius/radius/pkg/ucp/resources"
 	"github.com/project-radius/radius/pkg/ucp/store"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
@@ -515,7 +519,14 @@ func TestRunOperation_Timeout(t *testing.T) {
 	// set up mocks
 	tCtx.mockSC.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(testResourceObject, nil).AnyTimes()
 	tCtx.mockSC.EXPECT().Save(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
-	tCtx.mockSM.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+	tCtx.mockSM.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+		DoAndReturn(func(_ context.Context, _ resources.ID, _ uuid.UUID, state v1.ProvisioningState, _ *time.Time, opError *armerrors.ErrorDetails) error {
+			if state == v1.ProvisioningStateCanceled && strings.HasPrefix(opError.Message, "Operation (APPLICATIONS.CORE/ENVIRONMENTS|PUT) has timed out because it was processing longer than") &&
+				strings.HasPrefix(opError.Target, "/subscriptions/00000000-0000-0000-0000-000000000000") {
+				return nil
+			}
+			return errors.New("!!! failed to update status !!!")
+		}).Times(1)
 
 	testMessage := genTestMessage(uuid.New(), 10*time.Millisecond)
 	err := tCtx.testQueue.Enqueue(tCtx.ctx, testMessage)
diff --git a/test/functional/corerp/resources/testdata/corerp-azure-connection-database-service.bicep b/test/functional/corerp/resources/testdata/corerp-azure-connection-database-service.bicep
index 6351c2a9b..bb95d652b 100644
--- a/test/functional/corerp/resources/testdata/corerp-azure-connection-database-service.bicep
+++ b/test/functional/corerp/resources/testdata/corerp-azure-connection-database-service.bicep
@@ -1,9 +1,13 @@
 import radius as radius
 
 param magpieimage string = 'radiusdev.azurecr.io/magpiego:latest'
+
 param environment string
+
 param location string = resourceGroup().location
 
+param resourceIdentifier string = newGuid()
+
 resource app 'Applications.Core/applications@2022-03-15-privatepreview' = {
   name: 'corerp-azure-connection-database-service'
   location: 'global'
@@ -33,7 +37,7 @@ resource store 'Applications.Core/containers@2022-03-15-privatepreview' = {
 }
 
 resource databaseAccount 'Microsoft.DocumentDB/databaseAccounts@2020-04-01' = {
-  name: 'dbacc-${guid(resourceGroup().name)}'
+  name: 'dbacc-${resourceIdentifier}'
   location: location
   kind: 'MongoDB'
   properties: {
diff --git a/test/functional/corerp/resources/testdata/corerp-resources-dapr-pubsub-servicebus.bicep b/test/functional/corerp/resources/testdata/corerp-resources-dapr-pubsub-servicebus.bicep
index b67dd30a3..1afa402db 100644
--- a/test/functional/corerp/resources/testdata/corerp-resources-dapr-pubsub-servicebus.bicep
+++ b/test/functional/corerp/resources/testdata/corerp-resources-dapr-pubsub-servicebus.bicep
@@ -6,6 +6,8 @@ param environment string
 
 param location string = resourceGroup().location
 
+param resourceIdentifier string = newGuid()
+
 resource app 'Applications.Core/applications@2022-03-15-privatepreview' = {
   name: 'corerp-resources-dapr-pubsub-servicebus'
   location: location
@@ -58,7 +60,7 @@ resource pubsub 'Applications.Connector/daprPubSubBrokers@2022-03-15-privateprev
 }
 
 resource namespace 'Microsoft.ServiceBus/namespaces@2017-04-01' = {
-  name: 'daprns-${guid(resourceGroup().name)}'
+  name: 'daprns-${resourceIdentifier}'
   location: location
   tags: {
     radiustest: 'corerp-resources-dapr-pubsub-servicebus'
diff --git a/test/functional/corerp/resources/testdata/corerp-resources-dapr-statestore-tablestorage.bicep b/test/functional/corerp/resources/testdata/corerp-resources-dapr-statestore-tablestorage.bicep
index adfda55a3..98f097e86 100644
--- a/test/functional/corerp/resources/testdata/corerp-resources-dapr-statestore-tablestorage.bicep
+++ b/test/functional/corerp/resources/testdata/corerp-resources-dapr-statestore-tablestorage.bicep
@@ -6,6 +6,8 @@ param environment string
 
 param location string = resourceGroup().location
 
+param resourceIdentifier string = newGuid()
+
 resource app 'Applications.Core/applications@2022-03-15-privatepreview' = {
   name: 'corerp-resources-dapr-statestore-tablestorage'
   location: location
@@ -43,7 +45,7 @@ resource myapp 'Applications.Core/containers@2022-03-15-privatepreview' = {
 }
 
 resource account 'Microsoft.Storage/storageAccounts@2019-06-01' = {
-  name: 'dapr${uniqueString(resourceGroup().id, deployment().name)}'
+  name: 'dapr${resourceIdentifier}'
   location: location
   sku: {
     name: 'Standard_LRS'
diff --git a/test/functional/corerp/resources/testdata/corerp-resources-microsoft-sql.bicep b/test/functional/corerp/resources/testdata/corerp-resources-microsoft-sql.bicep
index 4f18a35f1..b2121551e 100644
--- a/test/functional/corerp/resources/testdata/corerp-resources-microsoft-sql.bicep
+++ b/test/functional/corerp/resources/testdata/corerp-resources-microsoft-sql.bicep
@@ -18,6 +18,8 @@ param adminUsername string = 'cooluser'
 @description('Specifies the SQL password.')
 param adminPassword string = 'p@ssw0rd'
 
+param resourceIdentifier string = newGuid()
+
 resource app 'Applications.Core/applications@2022-03-15-privatepreview' = {
   name: 'corerp-resources-microsoft-sql'
   location: location
@@ -61,7 +63,7 @@ resource db 'Applications.Connector/sqlDatabases@2022-03-15-privatepreview' = {
 }
 
 resource server 'Microsoft.Sql/servers@2021-02-01-preview' = {
-  name: 'mssql-${uniqueString(resourceGroup().id)}'
+  name: 'mssql-${resourceIdentifier}'
   location: location
   tags: {
     radiustest: 'corerp-resources-microsoft-sql'
diff --git a/test/functional/corerp/resources/testdata/corerp-resources-mongodb.bicep b/test/functional/corerp/resources/testdata/corerp-resources-mongodb.bicep
index c267f748d..587413514 100644
--- a/test/functional/corerp/resources/testdata/corerp-resources-mongodb.bicep
+++ b/test/functional/corerp/resources/testdata/corerp-resources-mongodb.bicep
@@ -6,6 +6,8 @@ param environment string
 
 param location string = resourceGroup().location
 
+param resourceIdentifier string = newGuid()
+
 resource app 'Applications.Core/applications@2022-03-15-privatepreview' = {
   name: 'corerp-resources-mongodb'
   location: 'global'
@@ -46,7 +48,7 @@ resource db 'Applications.Connector/mongoDatabases@2022-03-15-privatepreview' =
 }
 
 resource account 'Microsoft.DocumentDB/databaseAccounts@2020-04-01' = {
-  name: 'account-${guid(resourceGroup().name)}'
+  name: 'account-${resourceIdentifier}'
   location: location
   kind: 'MongoDB'
   tags: {