Repository: bacalhau-project/bacalhau (language: python)

Issue:
`bacalhau node list` returns error `failed request: invalid node type: nodeTypeUndefined`
### Bug Description
See title

### Expected Behavior
It lists the nodes

### Steps to Reproduce
1. install main
2. run a server
3. list nodes
4. see error

### Bacalhau Versions
- **Agent Version**:  `v1.3.1-rc1
- **CLI Client Version**: `v1.3.1-rc1`

### Host Environment
*Provide details about the environment where the bug occurred:*
- Operating System: linux
- CPU Architecture: x86



Discussion / hints:
I am working from the staging cluster which contains 4 nodes.. It appears there is an extra node (a 5th) in the store somewhere that is of type undefined causing these errors:
```
curl http://bootstrap.staging.bacalhau.org:1234/api/v1/orchestrator/nodes
{
  "NextToken": "",
  "Nodes": [
    {
      "Info": {
        "NodeID": "",
        "NodeType": "nodeTypeUndefined",
        "Labels": null,
        "BacalhauVersion": {
          "GitVersion": "",
          "GitCommit": "",
          "BuildDate": "0001-01-01T00:00:00Z",
          "GOOS": "",
          "GOARCH": ""
        }
      },
      "Membership": "",
      "Connection": "DISCONNECTED"
    },
    {
      "Info": {
        "NodeID": "QmRr9qPTe4mU7aS9faKnWgvn1NtXt36FT8YUULRPCn2f3K",
        "NodeType": "Compute",
        "Labels": {
          "Architecture": "amd64",
          "Operating-System": "linux",
          "git-lfs": "false",
          "owner": "bacalhau"
        },
        "ComputeNodeInfo": {
          "ExecutionEngines": [
            "docker",
            "wasm"
          ],
          "Publishers": [
            "s3",
            "local",
            "noop",
            "ipfs"
          ],
          "StorageSources": [
            "inline",
            "repoclone",
            "repoclonelfs",
            "s3",
            "ipfs",
            "urldownload"
          ],
          "MaxCapacity": {
            "CPU": 3.2,
            "Memory": 13406204723,
            "Disk": 83047314227
          },
          "QueueCapacity": {},
          "AvailableCapacity": {
            "CPU": 3.2,
            "Memory": 13406204723,
            "Disk": 83047314227
          },
          "MaxJobRequirements": {
            "CPU": 3.2,
            "Memory": 13406204723,
            "Disk": 83047314227
          },
          "RunningExecutions": 0,
          "EnqueuedExecutions": 0
        },
        "BacalhauVersion": {
          "GitVersion": "",
          "GitCommit": "",
          "BuildDate": "0001-01-01T00:00:00Z",
          "GOOS": "",
          "GOARCH": ""
        }
      },
      "Membership": "",
      "Connection": "CONNECTED"
    },
    {
      "Info": {
        "NodeID": "QmVHCeiLzhFJPCyCj5S1RTAk1vBEvxd8r5A6E4HyJGQtbJ",
        "NodeType": "Compute",
        "Labels": {
          "Architecture": "amd64",
          "Operating-System": "linux",
          "git-lfs": "false",
          "owner": "bacalhau"
        },
        "ComputeNodeInfo": {
          "ExecutionEngines": [
            "docker",
            "wasm"
          ],
          "Publishers": [
            "noop",
            "ipfs",
            "s3",
            "local"
          ],
          "StorageSources": [
            "inline",
            "repoclone",
            "repoclonelfs",
            "s3",
            "ipfs",
            "urldownload"
          ],
          "MaxCapacity": {
            "CPU": 3.2,
            "Memory": 13406204723,
            "Disk": 83046763724
          },
          "QueueCapacity": {},
          "AvailableCapacity": {
            "CPU": 3.2,
            "Memory": 13406204723,
            "Disk": 83046763724
          },
          "MaxJobRequirements": {
            "CPU": 3.2,
            "Memory": 13406204723,
            "Disk": 83046763724
          },
          "RunningExecutions": 0,
          "EnqueuedExecutions": 0
        },
        "BacalhauVersion": {
          "GitVersion": "",
          "GitCommit": "",
          "BuildDate": "0001-01-01T00:00:00Z",
          "GOOS": "",
          "GOARCH": ""
        }
      },
      "Membership": "",
      "Connection": "CONNECTED"
    },
    {
      "Info": {
        "NodeID": "Qma5yQAkEDWKBUZd3G4YRpvv5qBMpKvFywR7sqB34LB2Aw",
        "NodeType": "Compute",
        "Labels": {
          "Architecture": "amd64",
          "GPU-0": "Tesla-T4",
          "GPU-0-Memory": "15360-MiB",
          "Operating-System": "linux",
          "git-lfs": "false",
          "owner": "bacalhau"
        },
        "ComputeNodeInfo": {
          "ExecutionEngines": [
            "docker",
            "wasm"
          ],
          "Publishers": [
            "local",
            "noop",
            "ipfs",
            "s3"
          ],
          "StorageSources": [
            "repoclonelfs",
            "s3",
            "ipfs",
            "urldownload",
            "inline",
            "repoclone"
          ],
          "MaxCapacity": {
            "CPU": 3.2,
            "Memory": 12560636313,
            "Disk": 32934753075,
            "GPU": 1,
            "GPUs": [
              {
                "Index": 0,
                "Name": "Tesla T4",
                "Vendor": "NVIDIA",
                "Memory": 15360,
                "PCIAddress": ""
              }
            ]
          },
          "QueueCapacity": {},
          "AvailableCapacity": {
            "CPU": 3.2,
            "Memory": 12560636313,
            "Disk": 32934753075,
            "GPU": 1,
            "GPUs": [
              {
                "Index": 0,
                "Name": "Tesla T4",
                "Vendor": "NVIDIA",
                "Memory": 15360,
                "PCIAddress": ""
              }
            ]
          },
          "MaxJobRequirements": {
            "CPU": 3.2,
            "Memory": 12560636313,
            "Disk": 32934753075,
            "GPU": 1,
            "GPUs": [
              {
                "Index": 0,
                "Name": "Tesla T4",
                "Vendor": "NVIDIA",
                "Memory": 15360,
                "PCIAddress": ""
              }
            ]
          },
          "RunningExecutions": 0,
          "EnqueuedExecutions": 0
        },
        "BacalhauVersion": {
          "GitVersion": "",
          "GitCommit": "",
          "BuildDate": "0001-01-01T00:00:00Z",
          "GOOS": "",
          "GOARCH": ""
        }
      },
      "Membership": "",
      "Connection": "CONNECTED"
    },
    {
      "Info": {
        "NodeID": "QmafZ9oCXCJZX9Wt1nhrGS9FVVq41qhcBRSNWCkVhz3Nvv",
        "NodeType": "Compute",
        "Labels": {
          "Architecture": "amd64",
          "Operating-System": "linux",
          "git-lfs": "false",
          "owner": "bacalhau"
        },
        "ComputeNodeInfo": {
          "ExecutionEngines": [
            "docker",
            "wasm"
          ],
          "Publishers": [
            "s3",
            "local",
            "noop",
            "ipfs"
          ],
          "StorageSources": [
            "ipfs",
            "urldownload",
            "inline",
            "repoclone",
            "repoclonelfs",
            "s3"
          ],
          "MaxCapacity": {
            "CPU": 3.2,
            "Memory": 13406208000,
            "Disk": 79441883955
          },
          "QueueCapacity": {},
          "AvailableCapacity": {
            "CPU": 3.2,
            "Memory": 13406208000,
            "Disk": 79441883955
          },
          "MaxJobRequirements": {
            "CPU": 3.2,
            "Memory": 13406208000,
            "Disk": 79441883955
          },
          "RunningExecutions": 0,
          "EnqueuedExecutions": 0
        },
        "BacalhauVersion": {
          "GitVersion": "",
          "GitCommit": "",
          "BuildDate": "0001-01-01T00:00:00Z",
          "GOOS": "",
          "GOARCH": ""
        }
      },
      "Membership": "APPROVED",
      "Connection": "CONNECTED"
    }
  ]
}
```
I believe this may be an issue related to previous state in the clusters node store, as filtering for nodes that are connected work, but disconnected does not:
```
export BACALHAU_API_HOST=bootstrap.staging.bacalhau.org
frrist@cypress ~> bacalhau node list --filter-status=connected
 ID        TYPE     APPROVAL  STATUS     LABELS                                              CPU     MEMORY      DISK         GPU  
 QmRr9qPT  Compute            CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   77.3 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     77.3 GB      0    
 QmVHCeiL  Compute            CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   77.3 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     77.3 GB      0    
 Qma5yQAk  Compute            CONNECTED  Architecture=amd64 GPU-0-Memory=15360-MiB           3.2 /   11.7 GB /   30.7 GB /    1 /  
                                         GPU-0=Tesla-T4 Operating-System=linux               3.2     11.7 GB     30.7 GB      1    
                                         git-lfs=false owner=bacalhau                                                              
 QmafZ9oC  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   74.0 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     74.0 GB      0    

frrist@cypress ~> bacalhau node list --filter-status=disconnected
Error: failed request: invalid node type: nodeTypeUndefined
Usage:
  bacalhau node list [flags]

Flags:
      --filter-approval string   Filter nodes by approval. One of: ["approved" "pending" "rejected"]
      --filter-status string     Filter nodes by status. One of: ["connected" "disconnected"]
  -h, --help                     help for list
      --hide-header              do not print the column headers.
      --labels string            Filter nodes by labels. See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ for more information.
      --limit uint32             Limit the number of results returned
      --next-token string        Next token to use for pagination
      --no-style                 remove all styling from table output.
      --order-by string          Order results by a field. Valid fields are: id, type, available_cpu, available_memory, available_disk, available_gpu, status
      --order-reversed           Reverse the order of the results
      --output format            The output format for the command (one of ["table" "csv" "json" "yaml"]) (default table)
      --pretty                   Pretty print the output. Only applies to json and yaml output formats.
      --show strings             What column groups to show. Zero or more of: ["labels" "version" "features" "capacity"] (default [labels,capacity])
      --wide                     Print full values in the table results

Global Flags:
      --api-host string         The host for the client and server to communicate on (via REST).
                                Ignored if BACALHAU_API_HOST environment variable is set. (default "bootstrap.production.bacalhau.org")
      --api-port int            The port for the client and server to communicate on (via REST).
                                Ignored if BACALHAU_API_PORT environment variable is set. (default 1234)
      --cacert string           The location of a CA certificate file when self-signed certificates
                                	are used by the server
      --insecure                Enables TLS but does not verify certificates
      --log-mode logging-mode   Log format: 'default','station','json','combined','event' (default default)
      --repo string             path to bacalhau repo (default "/home/frrist/.bacalhau")
      --tls                     Instructs the client to use TLS

failed request: invalid node type: nodeTypeUndefined
```

I have performed the following operations on each node in the cluster to remove the invalid node from the state of the requester node:

**bacalhau-vm-stage-0 (requester+compute)**
```
systemctl stop bacalhau
rm /data/compute_store/QmafZ9oCXCJZX9Wt1nhrGS9FVVq41qhcBRSNWCkVhz3Nvv.registration.lock
rm /data/orchestrator_store/nats-store
systemctl start bacalhau
```

**bacalhau-vm-stage-1 (compute)**
```
systemctl stop bacalhau
rm compute_store/QmVHCeiLzhFJPCyCj5S1RTAk1vBEvxd8r5A6E4HyJGQtbJ.registration.lock
systemctl start bacalhau
```

**bacalhau-vm-stage-2 (compute)**
```
systemctl stop bacalhau
rm compute_store/QmRr9qPTe4mU7aS9faKnWgvn1NtXt36FT8YUULRPCn2f3K.registration.lock
systemctl start bacalhau
```

**bacalhau-vm-stage-3 (compute)**
```
systemctl stop bacalhau
rm compute_store/Qma5yQAkEDWKBUZd3G4YRpvv5qBMpKvFywR7sqB34LB2Aw.registration.lock
systemctl start bacalhau
```
 The node list command is now working as expected:
 ```
 frrist@cypress ~> bacalhau node list
 ID        TYPE     APPROVAL  STATUS     LABELS                                              CPU     MEMORY      DISK         GPU  
 QmRr9qPT  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   77.3 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     77.3 GB      0    
 QmVHCeiL  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   77.3 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     77.3 GB      0    
 Qma5yQAk  Compute  APPROVED  CONNECTED  Architecture=amd64 GPU-0-Memory=15360-MiB           3.2 /   11.7 GB /   30.7 GB /    1 /  
                                         GPU-0=Tesla-T4 Operating-System=linux               3.2     11.7 GB     30.7 GB      1    
                                         git-lfs=false owner=bacalhau                                                              
 QmafZ9oC  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   74.0 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     74.0 GB      0    

frrist@cypress ~> bacalhau node list --filter-status=connected
 ID        TYPE     APPROVAL  STATUS     LABELS                                              CPU     MEMORY      DISK         GPU  
 QmRr9qPT  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   77.3 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     77.3 GB      0    
 QmVHCeiL  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   77.3 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     77.3 GB      0    
 Qma5yQAk  Compute  APPROVED  CONNECTED  Architecture=amd64 GPU-0-Memory=15360-MiB           3.2 /   11.7 GB /   30.7 GB /    1 /  
                                         GPU-0=Tesla-T4 Operating-System=linux               3.2     11.7 GB     30.7 GB      1    
                                         git-lfs=false owner=bacalhau                                                              
 QmafZ9oC  Compute  APPROVED  CONNECTED  Architecture=amd64 Operating-System=linux           3.2 /   12.5 GB /   74.0 GB /    0 /  
                                         git-lfs=false owner=bacalhau                        3.2     12.5 GB     74.0 GB      0    

frrist@cypress ~> bacalhau node list --filter-status=disconnected
 ID  TYPE  APPROVAL  STATUS  LABELS  CPU  MEMORY  DISK  GPU 

```

Fix patch:
diff --git a/pkg/node/node.go b/pkg/node/node.go
index 388e874acc..7faf659d1f 100644
--- a/pkg/node/node.go
+++ b/pkg/node/node.go
@@ -230,7 +230,7 @@ func NewNode(
 				return nil, pkgerrors.Wrap(err, "failed to create NATS client for node info store")
 			}
 			nodeInfoStore, err := kvstore.NewNodeStore(ctx, kvstore.NodeStoreParams{
-				BucketName: kvstore.DefaultBucketName,
+				BucketName: kvstore.BucketNameCurrent,
 				Client:     natsClient.Client,
 			})
 			if err != nil {
diff --git a/pkg/routing/kvstore/kvstore.go b/pkg/routing/kvstore/kvstore.go
index f288adbb3c..0fc5b3a6da 100644
--- a/pkg/routing/kvstore/kvstore.go
+++ b/pkg/routing/kvstore/kvstore.go
@@ -17,7 +17,10 @@ import (
 )
 
 const (
-	DefaultBucketName = "nodes"
+	// BucketNameCurrent is the bucket name for bacalhau version v1.3.1 and beyond.
+	BucketNameCurrent = "node_v1"
+	// BucketNameV0 is the bucket name for bacalhau version v1.3.0 and below.
+	BucketNameV0 = "nodes"
 )
 
 type NodeStoreParams struct {
@@ -40,6 +43,11 @@ func NewNodeStore(ctx context.Context, params NodeStoreParams) (*NodeStore, erro
 	if bucketName == "" {
 		return nil, pkgerrors.New("bucket name is required")
 	}
+
+	if err := migrateJetStreamBucket(ctx, js, BucketNameV0, bucketName, migrateNodeInfoToNodeState); err != nil {
+		return nil, err
+	}
+
 	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
 		Bucket: bucketName,
 	})
diff --git a/pkg/routing/kvstore/migrations.go b/pkg/routing/kvstore/migrations.go
new file mode 100644
index 0000000000..ffee0819d9
--- /dev/null
+++ b/pkg/routing/kvstore/migrations.go
@@ -0,0 +1,114 @@
+package kvstore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/rs/zerolog/log"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+)
+
+func migrateNodeInfoToNodeState(entry jetstream.KeyValueEntry) ([]byte, error) {
+	var nodeinfo models.NodeInfo
+	if err := json.Unmarshal(entry.Value(), &nodeinfo); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal node info: %w", err)
+	}
+
+	nodestate := models.NodeState{
+		Info:       nodeinfo,
+		Membership: models.NodeMembership.PENDING,
+		Connection: models.NodeStates.DISCONNECTED,
+	}
+	migratedData, err := json.Marshal(nodestate)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal node state: %w", err)
+	}
+	return migratedData, nil
+}
+
+func migrateJetStreamBucket(
+	ctx context.Context,
+	js jetstream.JetStream,
+	from string,
+	to string,
+	migrateFunc func(entry jetstream.KeyValueEntry) ([]byte, error),
+) (retErr error) {
+	defer func() {
+		if retErr == nil {
+			if err := js.DeleteKeyValue(ctx, from); err != nil {
+				if errors.Is(err, jetstream.ErrBucketNotFound) {
+					// migration is successful since there isn't previous state to migrate from
+					retErr = nil
+				} else {
+					retErr = fmt.Errorf("NodeStore migration succeeded, but failed to remove old bucket: %w", err)
+				}
+			}
+		}
+	}()
+
+	fromKV, err := js.KeyValue(ctx, from)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrBucketNotFound) {
+			// migration is successful since there isn't previous state to migrate from
+			return nil
+		}
+		return fmt.Errorf("NodeStore migration failed: failed to open 'from' bucket: %w", err)
+	}
+
+	keys, err := fromKV.Keys(ctx)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrNoKeysFound) {
+			// if the store is empty the migration is successful as there isn't anything to migrate
+			return nil
+		}
+		return fmt.Errorf("NodeStore migration failed: failed to list store: %w", err)
+	}
+
+	start := time.Now()
+	log.Info().Str("from_bucket", from).Str("to_bucket", to).Msgf("Begin NodeStore migration")
+	toKV, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
+		Bucket: to,
+	})
+	if err != nil {
+		return fmt.Errorf("NodeStore migration failed: failed to open to bucket: %w", err)
+	}
+
+	for _, key := range keys {
+		// Check if the key exists in the 'to' bucket
+		_, err := toKV.Get(ctx, key)
+		if err == nil {
+			// Key already exists in the 'to' bucket, skip to the next key
+			continue
+		}
+		if !errors.Is(err, jetstream.ErrKeyNotFound) {
+			// An unexpected error occurred while checking the key in the 'to' bucket
+			return fmt.Errorf("NodeStore migration failed: failed to check key in 'to' bucket: %w", err)
+		}
+
+		// Read the entry from the 'from' bucket
+		entry, err := fromKV.Get(ctx, key)
+		if err != nil {
+			return fmt.Errorf("NodeStore migration failed: failed to read entry with key: %s: %w", key, err)
+		}
+
+		// Apply the migration function
+		migratedData, err := migrateFunc(entry)
+		if err != nil {
+			return fmt.Errorf("NodeStore migration failed: %w", err)
+		}
+
+		// Write the migrated data to the 'to' bucket
+		if _, err := toKV.Put(ctx, key, migratedData); err != nil {
+			return fmt.Errorf("NodeStore migration failed: failed to write migrated data to store: %w", err)
+		}
+	}
+
+	log.Info().Str("from_bucket", from).Str("to_bucket", to).Str("duration", time.Since(start).String()).
+		Msgf("Completed NodeStore migration")
+	return nil
+}


Test patch:
diff --git a/pkg/routing/kvstore/migration_test.go b/pkg/routing/kvstore/migration_test.go
new file mode 100644
index 0000000000..176f432699
--- /dev/null
+++ b/pkg/routing/kvstore/migration_test.go
@@ -0,0 +1,144 @@
+//go:build unit || !integration
+
+package kvstore_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/nats-io/nats-server/v2/server"
+	natsserver "github.com/nats-io/nats-server/v2/test"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/bacalhau-project/bacalhau/pkg/routing/kvstore"
+)
+
+type KVMigrationSuite struct {
+	suite.Suite
+	nats   *server.Server
+	client *nats.Conn
+	js     jetstream.JetStream
+}
+
+func (s *KVMigrationSuite) SetupTest() {
+	opts := &natsserver.DefaultTestOptions
+	opts.Port = TEST_PORT
+	opts.JetStream = true
+	opts.StoreDir = s.T().TempDir()
+
+	s.nats = natsserver.RunServer(opts)
+	var err error
+	s.client, err = nats.Connect(s.nats.Addr().String())
+	s.Require().NoError(err)
+
+	s.js, err = jetstream.New(s.client)
+	s.Require().NoError(err)
+}
+
+func (s *KVMigrationSuite) TearDownTest() {
+	s.nats.Shutdown()
+	s.client.Close()
+}
+
+func TestKVMigrationSuite(t *testing.T) {
+	suite.Run(t, new(KVMigrationSuite))
+}
+
+func (s *KVMigrationSuite) TestMigrationFromNodeInfoToNodeState() {
+	ctx := context.Background()
+
+	// Create 'from' bucket and populate it, simulating a requester on v130 with state to migrate.
+	fromKV, err := s.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: kvstore.BucketNameV0})
+	s.Require().NoError(err)
+
+	nodeInfos := []models.NodeInfo{
+		generateNodeInfo("node1", models.EngineDocker),
+		generateNodeInfo("node2", models.EngineWasm),
+		generateNodeInfo("node3", models.EngineDocker, models.EngineWasm),
+	}
+
+	// populate bucket with models.NodeInfo, these will be migrated to models.NodeState
+	for _, n := range nodeInfos {
+		data, err := json.Marshal(n)
+		s.Require().NoError(err)
+		_, err = fromKV.Put(ctx, n.ID(), data)
+		s.Require().NoError(err)
+	}
+
+	fromBucket := kvstore.BucketNameV0
+	toBucket := kvstore.BucketNameCurrent
+
+	// Open a NodeStore to trigger migration
+	ns, err := kvstore.NewNodeStore(ctx, kvstore.NodeStoreParams{
+		BucketName: toBucket,
+		Client:     s.client,
+	})
+	s.Require().NoError(err)
+
+	// Assert the migrated data is correct
+	for _, ni := range nodeInfos {
+		ns, err := ns.Get(ctx, ni.ID())
+		s.Require().NoError(err)
+		s.Equal(models.NodeStates.DISCONNECTED, ns.Connection)
+		s.Equal(models.NodeMembership.PENDING, ns.Membership)
+		s.Equal(ni, ns.Info)
+	}
+
+	// Assert the from bucket has been cleaned up
+	_, err = s.js.KeyValue(ctx, fromBucket)
+	s.Require().Equal(jetstream.ErrBucketNotFound, err)
+}
+
+func (s *KVMigrationSuite) TestMigrationStoreEmpty() {
+	ctx := context.Background()
+
+	// Create an empty 'from' bucket
+	_, err := s.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: kvstore.BucketNameV0})
+	s.Require().NoError(err)
+
+	fromBucket := kvstore.BucketNameV0
+	toBucket := kvstore.BucketNameCurrent
+
+	// Open a NodeStore to trigger migration, in this case there is a from bucket, but it's empty.
+	ns, err := kvstore.NewNodeStore(ctx, kvstore.NodeStoreParams{
+		BucketName: toBucket,
+		Client:     s.client,
+	})
+	s.Require().NoError(err)
+
+	// Assert the from bucket has been cleaned up
+	_, err = s.js.KeyValue(ctx, fromBucket)
+	s.Require().Contains(err.Error(), "bucket not found")
+
+	// Assert that no data was migrated since the from bucket was empty
+	resp, err := ns.List(ctx)
+	s.Require().NoError(err)
+	s.Require().Len(resp, 0)
+}
+
+func (s *KVMigrationSuite) TestMigrationStoreDNE() {
+	ctx := context.Background()
+
+	fromBucket := kvstore.BucketNameV0
+	toBucket := kvstore.BucketNameCurrent
+
+	// Open a NodeStore to trigger migration, in this case there isn't a from bucket to migrate from.
+	ns, err := kvstore.NewNodeStore(ctx, kvstore.NodeStoreParams{
+		BucketName: toBucket,
+		Client:     s.client,
+	})
+	s.Require().NoError(err)
+
+	// Assert the from bucket has been cleaned up
+	_, err = s.js.KeyValue(ctx, fromBucket)
+	s.Require().Contains(err.Error(), "bucket not found")
+
+	// Assert that no data was migrated since the from bucket DNE (does not exist)
+	resp, err := ns.List(ctx)
+	s.Require().NoError(err)
+	s.Require().Len(resp, 0)
+}