Repository: skupperproject/skupper (go)

Issue:
Podman site crash leaves setup in confusing state
# Description
There are possibly a few bugs and enhancements in this issue...
There is a way to cause a router crash when initialising a Podman Site that leaves the environment in a state that is difficult for a user to understand what is going on and how to get a clean state that another skupper init can be performed.

In this case the crash is caused by a new RHEL9 requirement for the CPU architecture. If running in a virtual machine then the defaults may not pass the actual host architecture through to the guest, causing the router to crash once skupper cli thinks it has been successfully started. The error the router crashes with is: **Fatal glibc error: CPU does not support x86-64-v2**. Configuring the hypervisor to pass the actual CPU architecture through stops the crash.

However, this crash has uncovered an issue related to the state that the host is left in when the router crashes. Other crash types will likely have the same issue.

At the heart of this issue is a lack of guidance to help a user get out of the mess.

# What did I do?
1. Built a virtual machine on Proxmix that has a CPU architectire of kvm64 (default)
2. Start RHSI:
    ```
    $ skupper init --site-name MASTER --platform podman --ingress-host 10.10.10.6
    It is recommended to enable lingering for bryon, otherwise Skupper may not start on boot.
    Skupper is now installed for user 'bryon'.  Use 'skupper status' to get more information.
    ````
3. After discovering that the router was not actually running I tried again:  
```
$ skupper init --site-name MASTER --platform podman --ingress-host 10.10.10.6
Skupper has already been initialized for user 'bryon'.
```
3. With no understanding of what has gone wrong I cleaned up the deployment and tried again   
```
$ skupper delete
Skupper is now removed for user 'bryon'.

$ skupper init --site-name MASTER --platform podman --ingress-host 10.10.10.15
Error: Error initializing Skupper - ingress port already bound :55671
```


# What happened
The router started and then crashed. No feedback was provided to the user.    
The logs from journalctl (attached) had two significant entries (see below and attached)
The router port remained locked until I rebooted the machine.


## Errors in system logs

**1. CPU Architecture** 
Nov 29 07:27:28 rhsi-2 skupper-controller-podman[1956]: Fatal glibc error: CPU does not support x86-64-v2

**2. Core Dump**
```
Nov 29 07:27:26 rhsi-2 systemd-coredump[1903]: Process 1879 (podman) of user 1000 dumped core.
                                               
                                               Module libbz2.so.1 from rpm bzip2-1.0.8-13.fc38.x86_64
                                               Module libsepol.so.2 from rpm libsepol-3.5-1.fc38.x86_64
                                               Module libpcre2-8.so.0 from rpm pcre2-10.42-1.fc38.1.x86_64
                                               Module libattr.so.1 from rpm attr-2.5.1-6.fc38.x86_64
                                               Module libacl.so.1 from rpm acl-2.3.1-6.fc38.x86_64
                                               Module libcrypt.so.2 from rpm libxcrypt-4.4.36-1.fc38.x86_64
                                               Module libeconf.so.0 from rpm libeconf-0.5.2-1.fc38.x86_64
                                               Module libsemanage.so.2 from rpm libsemanage-3.5-2.fc38.x86_64
                                               Module libselinux.so.1 from rpm libselinux-3.5-1.fc38.x86_64
                                               Module libaudit.so.1 from rpm audit-3.1.2-5.fc38.x86_64
                                               Module libseccomp.so.2 from rpm libseccomp-2.5.3-4.fc38.x86_64
                                               Module podman from rpm podman-4.7.2-1.fc38.x86_64
                                               Stack trace of thread 1881:
                                               #0  0x0000555c0ed3dde1 runtime.raise.abi0 (podman + 0x47dde1)
                                               #1  0x0000555c0ed1d54e runtime.sigfwdgo (podman + 0x45d54e)
                                               #2  0x0000555c0ed1bb47 runtime.sigtrampgo (podman + 0x45bb47)
                                               #3  0x0000555c0ed3e0e9 runtime.sigtramp.abi0 (podman + 0x47e0e9)
                                               #4  0x00007f6119f88bb0 __restore_rt (libc.so.6 + 0x3dbb0)
                                               #5  0x0000555c0ed3dde1 runtime.raise.abi0 (podman + 0x47dde1)
                                               #6  0x0000555c0ed05bed runtime.fatalpanic (podman + 0x445bed)
                                               #7  0x0000555c0ed052db runtime.gopanic (podman + 0x4452db)
                                               #8  0x0000555c0ed1cddd runtime.sigpanic (podman + 0x45cddd)
                                               #9  0x0000555c0f5e2ddb github.com/containers/podman/v4/pkg/errorhandling.CloseQuiet (podman + 0xd22ddb)
                                               #10 0x0000555c0fbfd906 github.com/containers/podman/v4/libpod.(*Runtime).setupRootlessPortMappingViaRLK.func1 (podman + 0x133d906)
                                               #11 0x0000555c0fbfd862 github.com/containers/podman/v4/libpod.(*Runtime).setupRootlessPortMappingViaRLK (podman + 0x133d862)
                                               #12 0x0000555c0fbcfd0f github.com/containers/podman/v4/libpod.(*Container).setupRootlessNetwork (podman + 0x130fd0f)
                                               #13 0x0000555c0fb9fa25 github.com/containers/podman/v4/libpod.(*Container).handleRestartPolicy (podman + 0x12dfa25)
                                               #14 0x0000555c0fb84f18 github.com/containers/podman/v4/libpod.(*Container).Cleanup (podman + 0x12c4f18)
                                               #15 0x0000555c0fce44de github.com/containers/podman/v4/pkg/domain/infra/abi.(*ContainerEngine).ContainerCleanup (podman + 0x14244de)
                                               #16 0x0000555c0fe494ed github.com/containers/podman/v4/cmd/podman/containers.cleanup (podman + 0x15894ed)
                                               #17 0x0000555c0f292722 github.com/spf13/cobra.(*Command).execute (podman + 0x9d2722)
                                               #18 0x0000555c0f292f9d github.com/spf13/cobra.(*Command).ExecuteC (podman + 0x9d2f9d)
                                               #19 0x0000555c0ff308ac main.Execute (podman + 0x16708ac)
                                               #20 0x0000555c0ff2ff1f main.main (podman + 0x166ff1f)
                                               #21 0x0000555c0ed08052 runtime.main (podman + 0x448052)
                                               #22 0x0000555c0ed3c461 runtime.goexit.abi0 (podman + 0x47c461)
                                               ELF object binary architecture: AMD x86-64
``` 

# What did I expect to happen
For the crash I would have expected skupper cli not to return until the router had properly stabilised.
On the second skupper init that failed I would have expected some help to guide the user as to how to clean up the environment

**Environment Details**
1. Use a hypervisor (proxmox) and set the CPU architecture to ```kvm64```.
2. Fedora 38 Workstation
3. Skupper release: 1.5
7. Skupper router version: quay.io/skupper/skupper-router      2.5.0             sha256:55f014d0fcf4b612eccf0f74cfb3cc298a52d529c5187b8a53ffe18b2f6a4a70

**Attachments**

Journalctl output
[joutnalctl.log](https://github.com/skupperproject/skupper/files/13518999/joutnalctl.log)


Pull request description:
Prevents podman site creation when unable to run the router image
Fixes #1298.

Patch:
diff --git a/pkg/domain/podman/site.go b/pkg/domain/podman/site.go
index 3af66872..4f025a3a 100644
--- a/pkg/domain/podman/site.go
+++ b/pkg/domain/podman/site.go
@@ -8,11 +8,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/skupperproject/skupper/api/types"
 	"github.com/skupperproject/skupper/client/container"
 	"github.com/skupperproject/skupper/client/podman"
 	"github.com/skupperproject/skupper/pkg/config"
 	"github.com/skupperproject/skupper/pkg/domain"
+	"github.com/skupperproject/skupper/pkg/images"
 	"github.com/skupperproject/skupper/pkg/qdr"
 	"github.com/skupperproject/skupper/pkg/utils"
 	"github.com/skupperproject/skupper/pkg/version"
@@ -428,6 +430,11 @@ func (s *SiteHandler) canCreate(site *Site) error {
 		return fmt.Errorf("You cannot use a remote podman endpoint - %w", err)
 	}
 
+	// Validating router container runs without errors
+	err = s.runTempContainer()
+	if err != nil {
+		return fmt.Errorf("site cannot be created: %s", err)
+	}
 	return nil
 }
 
@@ -902,3 +909,41 @@ func (s *SiteHandler) createPrometheusConfigFiles(site *Site) error {
 	}
 	return nil
 }
+
+func (s *SiteHandler) runTempContainer() error {
+	cli := s.cli
+	err := cli.ImagePull(images.GetRouterImageName())
+	if err != nil {
+		return err
+	}
+	tempName := fmt.Sprintf("skupper-temp-%s", uuid.NewString()[:5])
+	err = cli.ContainerCreate(
+		&container.Container{
+			Name:          tempName,
+			Image:         images.GetRouterImageName(),
+			RestartPolicy: "no",
+			Command:       []string{"skrouterd", "-version"},
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("unable to validate container creation: %s", err)
+	}
+	defer func() {
+		_ = cli.ContainerRemove(tempName)
+	}()
+	if err = cli.ContainerStart(tempName); err != nil {
+		return fmt.Errorf("error starting validation container: %s", err)
+	}
+	ci, err := cli.ContainerInspect(tempName)
+	if err != nil {
+		return fmt.Errorf("error inspecting validation container: %s", err)
+	}
+	if ci.ExitCode != 0 {
+		logs, err := cli.ContainerLogs(tempName)
+		if err != nil {
+			return fmt.Errorf("error executing validation container (unable to read logs) - exit code: %d", ci.ExitCode)
+		}
+		return fmt.Errorf("unable to run %s - reason: %s", ci.Image, logs)
+	}
+	return nil
+}
diff --git a/pkg/flow/flow_mem_driver.go b/pkg/flow/flow_mem_driver.go
index 76fed167..dfd9aa88 100644
--- a/pkg/flow/flow_mem_driver.go
+++ b/pkg/flow/flow_mem_driver.go
@@ -2436,18 +2436,24 @@ func (fc *FlowCollector) reconcileConnectorRecords() error {
 				}
 				for _, process := range fc.Processes {
 					if siteId == process.Parent {
-						if process.SourceHost != nil {
+						if process.SourceHost != nil && matchHost != nil {
 							if *matchHost == *process.SourceHost {
-								connector.ProcessId = &process.Identity
-								connector.Target = process.Name
-								process.connector = &connector.Identity
-								process.ProcessBinding = &Bound
-								fc.updateNetworkStatus()
-								log.Printf("COLLECTOR: Connector %s/%s associated to process %s\n", connector.Identity, *connector.Address, *process.Name)
-								delete(fc.connectorsToReconcile, connId)
 								found = true
-								break
 							}
+						} else if process.HostName != nil {
+							if *process.HostName == *connector.DestHost {
+								found = true
+							}
+						}
+						if found {
+							connector.ProcessId = &process.Identity
+							connector.Target = process.Name
+							process.connector = &connector.Identity
+							process.ProcessBinding = &Bound
+							fc.updateNetworkStatus()
+							log.Printf("COLLECTOR: Connector %s/%s associated to process %s\n", connector.Identity, *connector.Address, *process.Name)
+							delete(fc.connectorsToReconcile, connId)
+							break
 						}
 					}
 				}


Test patch:
diff --git a/pkg/flow/flow_mem_driver_test.go b/pkg/flow/flow_mem_driver_test.go
index a6eb2f6c..e6a32819 100644
--- a/pkg/flow/flow_mem_driver_test.go
+++ b/pkg/flow/flow_mem_driver_test.go
@@ -12,9 +12,12 @@ import (
 )
 
 func TestRecordGraphWithMetrics(t *testing.T) {
-	name := "skupper-site"
-	namespace := "skupper-public"
+	name := "skupper-site1"
+	namespace := "skupper-public1"
 	provider := "aws"
+	name2 := "skupper-site2"
+	namespace2 := "skupper-public2"
+	provider2 := "gke"
 	address := "tcp-go-echo"
 	address2 := "redis-cart"
 	address3 := "mongo"
@@ -31,6 +34,8 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 	connectorProcess1 := "process:1"
 	connectorProcess2 := "process:2"
 	processConnector := "connector:0"
+	destHost1 := "10.20.30.40"
+	destHost2 := "host.cloud.com"
 
 	sites := []SiteRecord{
 		{
@@ -43,6 +48,16 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			NameSpace: &namespace,
 			Provider:  &provider,
 		},
+		{
+			Base: Base{
+				RecType:   recordNames[Site],
+				Identity:  "site:1",
+				StartTime: uint64(time.Now().UnixNano()) / uint64(time.Microsecond),
+			},
+			Name:      &name2,
+			NameSpace: &namespace2,
+			Provider:  &provider2,
+		},
 	}
 	routers := []RouterRecord{
 		{
@@ -126,9 +141,10 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 				Parent:    "site:0",
 				StartTime: uint64(time.Now().UnixNano()) / uint64(time.Microsecond),
 			},
-			Name:      &processName2,
-			GroupName: &groupName1,
-			connector: &processConnector,
+			Name:       &processName2,
+			GroupName:  &groupName1,
+			connector:  &processConnector,
+			SourceHost: &destHost1,
 		},
 		{
 			Base: Base{
@@ -139,6 +155,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			},
 			Name:      &processName1,
 			GroupName: &groupName1,
+			HostName:  &destHost2,
 		},
 	}
 	listeners := []ListenerRecord{
@@ -149,8 +166,9 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 				Parent:    "router:0",
 				StartTime: uint64(time.Now().UnixNano()) / uint64(time.Microsecond),
 			},
-			Address:  &address,
-			Protocol: &protocol,
+			Address:   &address,
+			AddressId: &address,
+			Protocol:  &protocol,
 		},
 		{
 			Base: Base{
@@ -159,7 +177,9 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 				Parent:    "router:1",
 				StartTime: uint64(time.Now().UnixNano()) / uint64(time.Microsecond),
 			},
-			Protocol: &protocol,
+			Protocol:  &protocol,
+			Address:   &address,
+			AddressId: &address,
 		},
 		{
 			Base: Base{
@@ -183,6 +203,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			Address:   &address3,
 			Protocol:  &protocol,
 			ProcessId: &connectorProcess1,
+			DestHost:  &destHost1,
 		},
 		{
 			Base: Base{
@@ -203,6 +224,19 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			Protocol:  &protocol,
 			Address:   &address4,
 			ProcessId: &connectorProcess2,
+			DestHost:  &destHost2,
+		},
+		{
+			Base: Base{
+				RecType:   recordNames[Connector],
+				Identity:  "connector:2",
+				Parent:    "router:0",
+				StartTime: uint64(time.Now().UnixNano()) / uint64(time.Microsecond),
+			},
+			Protocol:  &protocol,
+			Address:   &address,
+			ProcessId: &connectorProcess1,
+			DestHost:  &destHost1,
 		},
 	}
 	flows := []FlowRecord{
@@ -227,6 +261,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			Trace:       &flowTrace,
 			ProcessName: &processName2,
 			Process:     &connectorProcess1,
+			SourceHost:  &destHost1,
 		},
 		{
 			Base: Base{
@@ -377,14 +412,29 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 		}
 		if f.Trace != nil {
 			trace := fc.annotateFlowTrace(&f)
-			assert.Equal(t, *trace, "skupper-site@skupper-site")
+			assert.Equal(t, *trace, "skupper-site1@skupper-site1")
 		}
 	}
 
 	fc.reconcileConnectorRecords()
 	fc.reconcileFlowRecords()
 
+	var addressId string
+	for _, address := range fc.VanAddresses {
+		if address.Name == "tcp-go-echo" {
+			addressId = address.Identity
+			break
+		}
+	}
+
+	var processGroupId string
+	for x, _ := range fc.ProcessGroups {
+		processGroupId = x
+		break
+	}
+
 	type test struct {
+		desc         string
 		recordType   int
 		method       string
 		url          string
@@ -396,6 +446,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 
 	testTable := []test{
 		{
+			desc:         "Get Site list",
 			recordType:   Site,
 			method:       "Get",
 			url:          "/",
@@ -405,6 +456,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(sites),
 		},
 		{
+			desc:         "Get Site item",
 			recordType:   Site,
 			method:       "Get",
 			url:          "/",
@@ -414,6 +466,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Site routers",
 			recordType:   Site,
 			method:       "Get",
 			url:          "/",
@@ -423,6 +476,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(routers),
 		},
 		{
+			desc:         "Get Site links",
 			recordType:   Site,
 			method:       "Get",
 			url:          "/",
@@ -432,6 +486,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(links),
 		},
 		{
+			desc:         "Get Site hosts",
 			recordType:   Site,
 			method:       "Get",
 			url:          "/",
@@ -441,6 +496,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(hosts),
 		},
 		{
+			desc:         "Get Site processes",
 			recordType:   Site,
 			method:       "Get",
 			url:          "/",
@@ -450,6 +506,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(processes),
 		},
 		{
+			desc:         "Get Host list",
 			recordType:   Host,
 			method:       "Get",
 			url:          "/",
@@ -459,6 +516,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(hosts),
 		},
 		{
+			desc:         "Get Host item",
 			recordType:   Host,
 			method:       "Get",
 			url:          "/",
@@ -468,6 +526,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Router list",
 			recordType:   Router,
 			method:       "Get",
 			url:          "/",
@@ -477,6 +536,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(routers),
 		},
 		{
+			desc:         "Get Router item",
 			recordType:   Router,
 			method:       "Get",
 			url:          "/",
@@ -486,6 +546,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Router flows",
 			recordType:   Router,
 			method:       "Get",
 			url:          "/",
@@ -495,6 +556,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 6,
 		},
 		{
+			desc:         "Get Router links",
 			recordType:   Router,
 			method:       "Get",
 			url:          "/",
@@ -504,6 +566,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Router listeners",
 			recordType:   Router,
 			method:       "Get",
 			url:          "/",
@@ -513,15 +576,17 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Router connectors",
 			recordType:   Router,
 			method:       "Get",
 			url:          "/",
 			params:       map[string]string{"sortBy": "identity.asc"},
 			vars:         map[string]string{"id": "router:0"},
 			name:         "connectors",
-			responseSize: 1,
+			responseSize: 2,
 		},
 		{
+			desc:         "Get Link list",
 			recordType:   Link,
 			method:       "Get",
 			url:          "/",
@@ -531,6 +596,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(links),
 		},
 		{
+			desc:         "Get Link item",
 			recordType:   Link,
 			method:       "Get",
 			url:          "/",
@@ -540,6 +606,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Listener list",
 			recordType:   Listener,
 			method:       "Get",
 			url:          "/",
@@ -549,6 +616,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(listeners),
 		},
 		{
+			desc:         "Get Listener item",
 			recordType:   Listener,
 			method:       "Get",
 			url:          "/",
@@ -558,6 +626,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Listener flows",
 			recordType:   Listener,
 			method:       "Get",
 			url:          "/",
@@ -567,15 +636,17 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 3,
 		},
 		{
+			desc:         "Get Connector list",
 			recordType:   Connector,
 			method:       "Get",
 			url:          "/",
 			params:       map[string]string{"sortBy": "identity.asc"},
 			vars:         map[string]string{},
 			name:         "list",
-			responseSize: 2,
+			responseSize: 3,
 		},
 		{
+			desc:         "Get Connector item",
 			recordType:   Connector,
 			method:       "Get",
 			url:          "/",
@@ -585,6 +656,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Connector flows",
 			recordType:   Connector,
 			method:       "Get",
 			url:          "/",
@@ -594,6 +666,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 3,
 		},
 		{
+			desc:         "Get Connector process",
 			recordType:   Connector,
 			method:       "Get",
 			url:          "/",
@@ -603,6 +676,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Address list",
 			recordType:   Address,
 			method:       "Get",
 			url:          "/",
@@ -612,15 +686,57 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(fc.VanAddresses),
 		},
 		{
-			recordType:   Connector,
+			desc:         "Get Address item",
+			recordType:   Address,
 			method:       "Get",
 			url:          "/",
 			params:       map[string]string{"sortBy": "identity.asc"},
-			vars:         map[string]string{"id": "connector:0"},
+			vars:         map[string]string{"id": addressId},
 			name:         "item",
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Address flows",
+			recordType:   Address,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{"id": addressId},
+			name:         "flows",
+			responseSize: 3,
+		},
+		{
+			desc:         "Get Address flowpairs",
+			recordType:   Address,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{"id": addressId},
+			name:         "flowpairs",
+			responseSize: 3,
+		},
+		{
+			desc:         "Get Address listeners",
+			recordType:   Address,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{"id": addressId},
+			name:         "listeners",
+			responseSize: 2,
+		},
+		{
+			desc:         "Get Address connectors",
+			recordType:   Address,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{"id": addressId},
+			name:         "connectors",
+			responseSize: 1,
+		},
+		{
+			desc:         "Get Flow list",
 			recordType:   Flow,
 			method:       "Get",
 			url:          "/",
@@ -630,6 +746,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(fc.Flows),
 		},
 		{
+			desc:         "Get Flow item",
 			recordType:   Flow,
 			method:       "Get",
 			url:          "/",
@@ -639,6 +756,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Flow process",
 			recordType:   Flow,
 			method:       "Get",
 			url:          "/",
@@ -648,6 +766,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Flowpair list",
 			recordType:   FlowPair,
 			method:       "Get",
 			url:          "/",
@@ -657,6 +776,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(fc.FlowPairs),
 		},
 		{
+			desc:         "Get Flowpair item",
 			recordType:   FlowPair,
 			method:       "Get",
 			url:          "/",
@@ -666,6 +786,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Process list",
 			recordType:   Process,
 			method:       "Get",
 			url:          "/",
@@ -675,6 +796,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: len(processes),
 		},
 		{
+			desc:         "Get Process item",
 			recordType:   Process,
 			method:       "Get",
 			url:          "/",
@@ -684,6 +806,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Process flows",
 			recordType:   Process,
 			method:       "Get",
 			url:          "/",
@@ -693,15 +816,17 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 3,
 		},
 		{
+			desc:         "Get Process addresses",
 			recordType:   Process,
 			method:       "Get",
 			url:          "/",
 			params:       map[string]string{"sortBy": "identity.asc"},
 			vars:         map[string]string{"id": "process:1"},
 			name:         "addresses",
-			responseSize: 1,
+			responseSize: 2,
 		},
 		{
+			desc:         "Get Process connector",
 			recordType:   Process,
 			method:       "Get",
 			url:          "/",
@@ -711,6 +836,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get ProcessGroup list",
 			recordType:   ProcessGroup,
 			method:       "Get",
 			url:          "/",
@@ -720,6 +846,27 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get ProcessGroup item",
+			recordType:   ProcessGroup,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{"id": processGroupId},
+			name:         "item",
+			responseSize: 1,
+		},
+		{
+			desc:         "Get ProcessGroup processes",
+			recordType:   ProcessGroup,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{"id": processGroupId},
+			name:         "processes",
+			responseSize: 3,
+		},
+		{
+			desc:         "Get SitePair list",
 			recordType:   SitePair,
 			method:       "Get",
 			url:          "/",
@@ -729,6 +876,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get SitePair item",
 			recordType:   SitePair,
 			method:       "Get",
 			url:          "/",
@@ -738,6 +886,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get ProcessGroupPair list",
 			recordType:   ProcessGroupPair,
 			method:       "Get",
 			url:          "/",
@@ -747,6 +896,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get ProcessPair list",
 			recordType:   ProcessPair,
 			method:       "Get",
 			url:          "/",
@@ -756,6 +906,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get ProcessPair item",
 			recordType:   ProcessPair,
 			method:       "Get",
 			url:          "/",
@@ -765,6 +916,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			responseSize: 1,
 		},
 		{
+			desc:         "Get Collector list",
 			recordType:   Collector,
 			method:       "Get",
 			url:          "/",
@@ -773,8 +925,21 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 			name:         "list",
 			responseSize: 1,
 		},
+		{
+			desc:         "Get Collector item",
+			recordType:   Collector,
+			method:       "Get",
+			url:          "/",
+			params:       map[string]string{"sortBy": "identity.asc"},
+			vars:         map[string]string{},
+			name:         "item",
+			responseSize: 1,
+		},
 	}
 
+	fc.reconcileConnectorRecords()
+	fc.reconcileFlowRecords()
+
 	var payload Payload
 	for _, test := range testTable {
 		req, _ := http.NewRequest(test.method, test.url, nil)
@@ -788,7 +953,7 @@ func TestRecordGraphWithMetrics(t *testing.T) {
 		assert.Assert(t, err)
 		err = json.Unmarshal([]byte(*resp), &payload)
 		assert.Assert(t, err)
-		assert.Equal(t, test.responseSize, payload.Count)
+		assert.Equal(t, test.responseSize, payload.Count, test.desc)
 	}
 
 	endTime := uint64(time.Now().UnixNano()) / uint64(time.Microsecond)