Repository: superfly/flyctl
Pull request #3840 (2024-08-13 21:39:47)

Dealing with unreachable machines on dead hosts
When a host is dead or unreachable, there are certain machine data like its config that can't be fully rebuilt. That prevents operations like listing from showing useful data, they may even panic.

The changes not only tries to cover the case of reporting machine status as much as possible, but also allows `fly deploy` to replace the machine(s) as long as there are no volumes attached.

---

### Documentation

- [ ] Fresh Produce
- [ ] In superfly/docs, or asked for help from docs team
- [ ] n/a



Patch:
diff --git a/go.mod b/go.mod
index f9c457fb8..06749e4bb 100644
--- a/go.mod
+++ b/go.mod
@@ -72,7 +72,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
-	github.com/superfly/fly-go v0.1.22
+	github.com/superfly/fly-go v0.1.25
 	github.com/superfly/graphql v0.2.4
 	github.com/superfly/lfsc-go v0.1.1
 	github.com/superfly/macaroon v0.2.14-0.20240702184853-b8ac52a1fc77
@@ -84,7 +84,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.28.0
 	go.opentelemetry.io/otel/trace v1.28.0
 	golang.org/x/crypto v0.26.0
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa
 	golang.org/x/mod v0.20.0
 	golang.org/x/net v0.28.0
 	golang.org/x/sync v0.8.0
@@ -254,7 +254,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/tools v0.23.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
diff --git a/go.sum b/go.sum
index 9306525e5..584607d5e 100644
--- a/go.sum
+++ b/go.sum
@@ -621,8 +621,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/superfly/fly-go v0.1.22 h1:Hkqak+gX4SWfhwt9yGs5CsP6f3I5Q7ZMZUyDgaDVZ6g=
-github.com/superfly/fly-go v0.1.22/go.mod h1:JQke/BwoZqrWurqYkypSlcSo7bIUgCI3eVnqMC6AUj0=
+github.com/superfly/fly-go v0.1.25 h1:h3kfSXREYRSlY1y5bKxxrql+nIvtEE2C6Wr0n1lg70o=
+github.com/superfly/fly-go v0.1.25/go.mod h1:JQke/BwoZqrWurqYkypSlcSo7bIUgCI3eVnqMC6AUj0=
 github.com/superfly/graphql v0.2.4 h1:Av8hSk4x8WvKJ6MTnEwrLknSVSGPc7DWpgT3z/kt3PU=
 github.com/superfly/graphql v0.2.4/go.mod h1:CVfDl31srm8HnJ9udwLu6hFNUW/P6GUM2dKcG1YQ8jc=
 github.com/superfly/lfsc-go v0.1.1 h1:dGjLgt81D09cG+aR9lJZIdmonjZSR5zYCi7s54+ZU2Q=
@@ -707,8 +707,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
 golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
 golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
-golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI=
+golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -823,8 +823,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
-golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/internal/command/deploy/machines.go b/internal/command/deploy/machines.go
index f879abf3c..79b7f5a7f 100644
--- a/internal/command/deploy/machines.go
+++ b/internal/command/deploy/machines.go
@@ -449,7 +449,8 @@ func (md *machineDeployment) validateVolumeConfig() error {
 			needsVol := map[string][]string{}
 
 			for _, m := range ms {
-				if mntDst == "" && len(m.Config.Mounts) != 0 {
+				mConfig := m.GetConfig()
+				if mntDst == "" && len(mConfig.Mounts) != 0 {
 					// TODO: Detaching a volume from a machine is possible, but it usually means a missconfiguration.
 					// We should show a warning and ask the user for confirmation and let it happen instead of failing here.
 					return fmt.Errorf(
@@ -459,13 +460,13 @@ func (md *machineDeployment) validateVolumeConfig() error {
 					)
 				}
 
-				if mntDst != "" && len(m.Config.Mounts) == 0 {
+				if mntDst != "" && len(mConfig.Mounts) == 0 {
 					// Attaching a volume to an existing machine is not possible, but we replace the machine
 					// by another running on the same zone than the volume.
 					needsVol[mntSrc] = append(needsVol[mntSrc], m.Region)
 				}
 
-				if mms := m.Config.Mounts; len(mms) > 0 && mntSrc != "" && mms[0].Name != "" && mntSrc != mms[0].Name {
+				if mms := mConfig.Mounts; len(mms) > 0 && mntSrc != "" && mms[0].Name != "" && mntSrc != mms[0].Name {
 					// TODO: Changed the attached volume to an existing machine is not possible, but it could replace the machine
 					// by another running on the same zone than the new volume.
 					return fmt.Errorf(
diff --git a/internal/command/deploy/machines_launchinput.go b/internal/command/deploy/machines_launchinput.go
index 5a84bbb59..564f323cd 100644
--- a/internal/command/deploy/machines_launchinput.go
+++ b/internal/command/deploy/machines_launchinput.go
@@ -67,9 +67,15 @@ func (md *machineDeployment) launchInputForLaunch(processGroup string, guest *fl
 func (md *machineDeployment) launchInputForUpdate(origMachineRaw *fly.Machine) (*fly.LaunchMachineInput, error) {
 	mID := origMachineRaw.ID
 	machineShouldBeReplaced := dedicatedHostIdMismatch(origMachineRaw, md.appConfig)
-	processGroup := origMachineRaw.Config.ProcessGroup()
 
-	mConfig, err := md.appConfig.ToMachineConfig(processGroup, origMachineRaw.Config)
+	oConfig := origMachineRaw.GetConfig()
+	if origMachineRaw.HostStatus != fly.HostStatusOk {
+		machineShouldBeReplaced = true
+	}
+
+	processGroup := oConfig.ProcessGroup()
+
+	mConfig, err := md.appConfig.ToMachineConfig(processGroup, oConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -83,7 +89,8 @@ func (md *machineDeployment) launchInputForUpdate(origMachineRaw *fly.Machine) (
 	//   * The only allowed in-place operation is to update its destination mount path
 	//   * The other option is to force a machine replacement to remove or attach a different volume
 	mMounts := mConfig.Mounts
-	oMounts := origMachineRaw.Config.Mounts
+	oMounts := oConfig.Mounts
+
 	if len(oMounts) != 0 {
 		var latestExtendThresholdPercent, latestAddSizeGb, latestSizeGbLimit int
 		if len(mMounts) > 0 {
@@ -144,13 +151,23 @@ func (md *machineDeployment) launchInputForUpdate(origMachineRaw *fly.Machine) (
 		machineShouldBeReplaced = true
 	}
 
+	if origMachineRaw.HostStatus != fly.HostStatusOk && len(oMounts) > 0 && len(mMounts) > 0 && oMounts[0].Volume == mMounts[0].Volume {
+		// We are attempting to replace an unreachable machine but reusing the volume id that ties it to the dead host
+		// TODO: Link to recovery instructions to manually create a new volume, empty or from snapshot, an only then recreate the machine.
+		return nil, fmt.Errorf(
+			"machine '%s' requires manual intervention, it can't be automatically replaced because its volume '%s' is on an unreachable host",
+			mID,
+			oMounts[0].Volume,
+		)
+	}
+
 	// If this is a standby machine that now has a service, then clear
 	// the standbys list.
 	if len(mConfig.Services) > 0 && len(mConfig.Standbys) > 0 {
 		mConfig.Standbys = nil
 	}
 
-	if hdid := md.appConfig.HostDedicationID; hdid != "" && hdid != origMachineRaw.Config.Guest.HostDedicationID {
+	if hdid := md.appConfig.HostDedicationID; hdid != "" && hdid != oConfig.Guest.HostDedicationID {
 		if len(oMounts) > 0 && len(mMounts) > 0 {
 			// Attempting to rellocate a machine with a volume attached to a different host
 			return nil, fmt.Errorf("can't rellocate machine '%s' to dedication id '%s' because it has an attached volume."+
diff --git a/internal/command/deploy/plan.go b/internal/command/deploy/plan.go
index d95f8760f..a66e45b0b 100644
--- a/internal/command/deploy/plan.go
+++ b/internal/command/deploy/plan.go
@@ -387,6 +387,11 @@ func (md *machineDeployment) acquireLeases(ctx context.Context, machineTuples []
 			}
 			sl := machToLogger.getLoggerFromID(machine.ID)
 
+			if machine.HostStatus == fly.HostStatusUnreachable {
+				sl.LogStatus(statuslogger.StatusRunning, fmt.Sprintf("Skipped lease for unreachable machine %s", machine.ID))
+				return nil
+			}
+
 			if machine.LeaseNonce != "" {
 				sl.LogStatus(statuslogger.StatusRunning, fmt.Sprintf("Already have lease for %s", machine.ID))
 				return nil
diff --git a/internal/command/machine/clone.go b/internal/command/machine/clone.go
index f51e0a84f..148c20674 100644
--- a/internal/command/machine/clone.go
+++ b/internal/command/machine/clone.go
@@ -96,6 +96,11 @@ func runMachineClone(ctx context.Context) (err error) {
 	if err != nil {
 		return err
 	}
+
+	if source.HostStatus != fly.HostStatusOk {
+		return fmt.Errorf("the machine is on an unreachable host, try again later")
+	}
+
 	flapsClient := flapsutil.ClientFromContext(ctx)
 
 	var vol *fly.Volume
diff --git a/internal/command/machine/destroy.go b/internal/command/machine/destroy.go
index 336187c6c..5693c7cbb 100644
--- a/internal/command/machine/destroy.go
+++ b/internal/command/machine/destroy.go
@@ -57,16 +57,21 @@ This command requires a machine to be in a stopped or suspended state unless the
 
 func runMachineDestroy(ctx context.Context) (err error) {
 	ctx, err = buildContextFromAppName(ctx, appconfig.NameFromContext(ctx))
+	if err != nil {
+		return err
+	}
+
+	var machinesToBeDeleted []*fly.Machine
 	image := strings.TrimSpace(flag.GetString(ctx, "image"))
-	ids := []string{}
 
-	if image != "" {
+	switch {
+	case image != "":
 		machines, err := flapsutil.ClientFromContext(ctx).ListActive(ctx)
 		if err != nil {
 			return err
 		}
 
-		machinesToBeDeleted := []*fly.Machine{}
+		var ids []string
 		for _, machine := range machines {
 			if machine.ImageRefWithVersion() == image {
 				machinesToBeDeleted = append(machinesToBeDeleted, machine)
@@ -74,18 +79,11 @@ func runMachineDestroy(ctx context.Context) (err error) {
 			}
 		}
 
-		if len(machinesToBeDeleted) == 0 {
-			fmt.Fprint(iostreams.FromContext(ctx).Out, "No machine to destroy, exiting\n")
-			return nil
-		}
-
-		machines, release, err := mach.AcquireLeases(ctx, machinesToBeDeleted)
-		defer release()
-		if err != nil {
-			return err
-		}
-
-		confirmed, err := prompt.Confirm(ctx, fmt.Sprintf("%d Machines (%s) will be destroyed, continue?", len(machines), strings.Join(ids, ",")))
+		confirmed, err := prompt.Confirm(ctx,
+			fmt.Sprintf("%d Machines (%s) will be destroyed, continue?",
+				len(machines),
+				strings.Join(ids, ","),
+			))
 		if err != nil {
 			return err
 		}
@@ -93,47 +91,39 @@ func runMachineDestroy(ctx context.Context) (err error) {
 			return nil
 		}
 
-		for _, machine := range machines {
-			err = singleDestroyRun(ctx, machine)
-			if err != nil {
-				return err
-			}
-		}
-
-	} else if len(flag.Args(ctx)) == 0 {
-		machine, ctx, err := selectOneMachine(ctx, "", "", false)
-		if err != nil {
-			return err
-		}
-		machine, release, err := mach.AcquireLease(ctx, machine)
-		defer release()
+	case len(flag.Args(ctx)) == 0:
+		machine, newCtx, err := selectOneMachine(ctx, "", "", false)
 		if err != nil {
 			return err
 		}
+		ctx = newCtx
+		machinesToBeDeleted = append(machinesToBeDeleted, machine)
 
-		err = singleDestroyRun(ctx, machine)
-		if err != nil {
-			return err
-		}
-	} else {
-		machines, ctx, err := selectManyMachines(ctx, flag.Args(ctx))
+	default:
+		machines, newCtx, err := selectManyMachines(ctx, flag.Args(ctx))
 		if err != nil {
 			return err
 		}
+		ctx = newCtx
+		machinesToBeDeleted = append(machinesToBeDeleted, machines...)
+	}
+
+	if len(machinesToBeDeleted) == 0 {
+		fmt.Fprint(iostreams.FromContext(ctx).Out, "No machine to destroy, exiting\n")
+		return nil
+	}
+
+	machines, release, err := mach.AcquireLeases(ctx, machinesToBeDeleted)
+	defer release()
+	if err != nil {
+		return err
+	}
 
-		machines, release, err := mach.AcquireLeases(ctx, machines)
-		defer release()
+	for _, machine := range machines {
+		err = singleDestroyRun(ctx, machine)
 		if err != nil {
 			return err
 		}
-
-		for _, machine := range machines {
-			err = singleDestroyRun(ctx, machine)
-			if err != nil {
-				return err
-			}
-		}
-
 	}
 
 	return nil
diff --git a/internal/command/machine/list.go b/internal/command/machine/list.go
index 0ee24e2e2..50ca8beae 100644
--- a/internal/command/machine/list.go
+++ b/internal/command/machine/list.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/samber/lo"
 	"github.com/spf13/cobra"
+	fly "github.com/superfly/fly-go"
 	"github.com/superfly/fly-go/flaps"
 	"github.com/superfly/flyctl/internal/appconfig"
 	"github.com/superfly/flyctl/internal/command"
@@ -114,7 +115,7 @@ func runMachineList(ctx context.Context) (err error) {
 			}
 
 			note := ""
-			unreachable := machine.HostStatus == "unreachable"
+			unreachable := machine.HostStatus != fly.HostStatusOk
 			if unreachable {
 				unreachableMachines = true
 				note = "*"
diff --git a/internal/command/machine/status.go b/internal/command/machine/status.go
index 3e7bc98f8..2a27839cf 100644
--- a/internal/command/machine/status.go
+++ b/internal/command/machine/status.go
@@ -93,9 +93,12 @@ func runMachineStatus(ctx context.Context) (err error) {
 		checksSummary = fmt.Sprintf("%d/%d", checksPassing, checksTotal)
 	}
 
+	mConfig := machine.GetConfig()
+
 	fmt.Fprintf(io.Out, "Machine ID: %s\n", machine.ID)
 	fmt.Fprintf(io.Out, "Instance ID: %s\n", machine.InstanceID)
 	fmt.Fprintf(io.Out, "State: %s\n", machine.State)
+	fmt.Fprintf(io.Out, "HostStatus: %s\n", machine.HostStatus)
 	fmt.Fprintf(io.Out, "\n")
 
 	obj := [][]string{
@@ -108,28 +111,28 @@ func runMachineStatus(ctx context.Context) (err error) {
 			machine.PrivateIP,
 			machine.Region,
 			machine.ProcessGroup(),
-			fmt.Sprint(machine.Config.Guest.CPUKind),
-			fmt.Sprint(machine.Config.Guest.CPUs),
-			fmt.Sprint(machine.Config.Guest.MemoryMB),
+			fmt.Sprint(mConfig.Guest.CPUKind),
+			fmt.Sprint(mConfig.Guest.CPUs),
+			fmt.Sprint(mConfig.Guest.MemoryMB),
 			machine.CreatedAt,
 			machine.UpdatedAt,
-			optJsonStrings(machine.Config.Init.Entrypoint),
-			optJsonStrings(machine.Config.Init.Cmd),
+			optJsonStrings(mConfig.Init.Entrypoint),
+			optJsonStrings(mConfig.Init.Cmd),
 		},
 	}
 
 	var cols []string = []string{"ID", "Instance ID", "State", "Image", "Name", "Private IP", "Region", "Process Group", "CPU Kind", "vCPUs", "Memory", "Created", "Updated", "Entrypoint", "Command"}
 
-	if len(machine.Config.Mounts) > 0 {
+	if len(mConfig.Mounts) > 0 {
 		cols = append(cols, "Volume")
-		obj[0] = append(obj[0], machine.Config.Mounts[0].Volume)
+		obj[0] = append(obj[0], mConfig.Mounts[0].Volume)
 	}
 
 	if err = render.VerticalTable(io.Out, "VM", obj, cols...); err != nil {
 		return
 	}
 
-	if machine.Config.Metadata["fly-managed-postgres"] == "true" {
+	if mConfig.Metadata["fly-managed-postgres"] == "true" {
 		obj := [][]string{
 			{
 				roleOutput,
@@ -172,7 +175,7 @@ func runMachineStatus(ctx context.Context) (err error) {
 
 	if flag.GetBool(ctx, "display-config") {
 		var prettyConfig []byte
-		prettyConfig, err = json.MarshalIndent(machine.Config, "", "  ")
+		prettyConfig, err = json.MarshalIndent(mConfig, "", "  ")
 
 		if err != nil {
 			return err
diff --git a/internal/command/machine/update.go b/internal/command/machine/update.go
index 2ac5f41e7..7fd522cc0 100644
--- a/internal/command/machine/update.go
+++ b/internal/command/machine/update.go
@@ -88,6 +88,10 @@ func runUpdate(ctx context.Context) (err error) {
 	}
 	appName := appconfig.NameFromContext(ctx)
 
+	if machine.HostStatus != fly.HostStatusOk {
+		return fmt.Errorf("the machine is on an unreachable host, try again later")
+	}
+
 	// Acquire lease
 	machine, releaseLeaseFunc, err := mach.AcquireLease(ctx, machine)
 	defer releaseLeaseFunc()
diff --git a/internal/command/status/machines.go b/internal/command/status/machines.go
index 2cb3c8e66..76f49d186 100644
--- a/internal/command/status/machines.go
+++ b/internal/command/status/machines.go
@@ -19,28 +19,24 @@ import (
 	"github.com/superfly/flyctl/iostreams"
 )
 
-func getFromMetadata(m *fly.Machine, key string) string {
-	if m.Config != nil && m.Config.Metadata != nil {
-		return m.Config.Metadata[key]
-	}
-
-	return ""
-}
-
 func getProcessgroup(m *fly.Machine) string {
 	name := m.ProcessGroup()
 	if name == "" {
 		name = "<default>"
 	}
 
-	if len(m.Config.Standbys) > 0 {
+	if len(m.GetConfig().Standbys) > 0 {
 		name += "†"
 	}
+
+	if m.HostStatus != fly.HostStatusOk {
+		name += "💀"
+	}
 	return name
 }
 
 func getReleaseVersion(m *fly.Machine) string {
-	return getFromMetadata(m, fly.MachineConfigMetadataKeyFlyReleaseVersion)
+	return m.GetMetadataByKey(fly.MachineConfigMetadataKeyFlyReleaseVersion)
 }
 
 // getImage returns the image on the most recent machine released under an app.
@@ -175,14 +171,19 @@ func RenderMachineStatus(ctx context.Context, app *fly.AppCompact, out io.Writer
 
 	if len(managed) > 0 {
 		hasStandbys := false
+		hasNotOk := false
 		rows := [][]string{}
 		for _, machine := range managed {
-			if len(machine.Config.Standbys) > 0 {
+			mConfig := machine.GetConfig()
+			if len(mConfig.Standbys) > 0 {
 				hasStandbys = true
 			}
+			if machine.HostStatus != fly.HostStatusOk {
+				hasNotOk = true
+			}
 			var role string
 
-			if v := machine.Config.Metadata["role"]; v != "" {
+			if v := mConfig.Metadata["role"]; v != "" {
 				role = v
 			}
 			rows = append(rows, []string{
@@ -206,9 +207,15 @@ func RenderMachineStatus(ctx context.Context, app *fly.AppCompact, out io.Writer
 			return err
 		}
 
+		if hasStandbys || hasNotOk {
+			fmt.Fprint(out, "Notes:\n")
+		}
 		if hasStandbys {
 			fmt.Fprintf(out, "  † Standby machine (it will take over only in case of host hardware failure)\n")
 		}
+		if hasNotOk {
+			fmt.Fprintf(out, "  💀 The machine's host is unreachable\n")
+		}
 	}
 
 	if len(unmanaged) > 0 {
diff --git a/internal/machine/leasable_machine.go b/internal/machine/leasable_machine.go
index 86909a7f1..cdab8c815 100644
--- a/internal/machine/leasable_machine.go
+++ b/internal/machine/leasable_machine.go
@@ -121,15 +121,12 @@ func (lm *leasableMachine) Cordon(ctx context.Context) error {
 }
 
 func (lm *leasableMachine) FormattedMachineId() string {
-	res := lm.Machine().ID
-	if lm.Machine().Config.Metadata == nil {
-		return res
+	m := lm.Machine()
+	processGroup := m.ProcessGroup()
+	if processGroup == "" || m.IsFlyAppsReleaseCommand() || m.IsFlyAppsConsole() {
+		return m.ID
 	}
-	procGroup := lm.Machine().ProcessGroup()
-	if procGroup == "" || lm.Machine().IsFlyAppsReleaseCommand() || lm.Machine().IsFlyAppsConsole() {
-		return res
-	}
-	return fmt.Sprintf("%s [%s]", res, procGroup)
+	return fmt.Sprintf("%s [%s]", m.ID, processGroup)
 }
 
 func (lm *leasableMachine) logStatusWaiting(ctx context.Context, desired string) {
diff --git a/internal/machine/lease.go b/internal/machine/lease.go
index fa9558c5f..9c0e3b9d9 100644
--- a/internal/machine/lease.go
+++ b/internal/machine/lease.go
@@ -35,6 +35,11 @@ func AcquireLeases(ctx context.Context, machines []*fly.Machine) ([]*fly.Machine
 	for _, m := range machines {
 		m := m
 		acquirePool.Go(func() (*fly.Machine, error) {
+			// Skip leasing for unreachable machines
+			if m.HostStatus != fly.HostStatusOk {
+				return m, nil
+			}
+
 			m, _, err := AcquireLease(ctx, m)
 			return m, err
 		})


Test patch:
diff --git a/internal/appconfig/machines_test.go b/internal/appconfig/machines_test.go
index 37ae04dce..8dd14be8a 100644
--- a/internal/appconfig/machines_test.go
+++ b/internal/appconfig/machines_test.go
@@ -342,6 +342,7 @@ func TestToTestMachineConfigWKillInfoNoImageAndOrigMachineKillInfo(t *testing.T)
 	}
 
 	origMachine := &fly.Machine{
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Image: "nginx",
 			StopConfig: &fly.StopConfig{
@@ -394,6 +395,7 @@ func TestToTestMachineConfigNoImageAndOrigMachineKillInfo(t *testing.T) {
 	}
 
 	origMachine := &fly.Machine{
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Image: "nginx",
 			StopConfig: &fly.StopConfig{
@@ -445,8 +447,9 @@ func TestToTestMachineConfigWTestMachine(t *testing.T) {
 
 	check := cfg.HTTPService.MachineChecks[0]
 	machine := &fly.Machine{
-		ImageRef:  fly.MachineImageRef{},
-		PrivateIP: "1.2.3.4",
+		HostStatus: fly.HostStatusOk,
+		ImageRef:   fly.MachineImageRef{},
+		PrivateIP:  "1.2.3.4",
 		Config: &fly.MachineConfig{
 			Env: map[string]string{
 				"BAR": "BAZ",
diff --git a/internal/command/deploy/machines_launchinput_test.go b/internal/command/deploy/machines_launchinput_test.go
index 31e04409f..af57eb555 100644
--- a/internal/command/deploy/machines_launchinput_test.go
+++ b/internal/command/deploy/machines_launchinput_test.go
@@ -55,9 +55,10 @@ func Test_launchInputFor_Basic(t *testing.T) {
 	md.releaseVersion = 4
 
 	origMachineRaw := &fly.Machine{
-		ID:     "ab1234567890",
-		Region: li.Region,
-		Config: helpers.Clone(li.Config),
+		ID:         "ab1234567890",
+		Region:     li.Region,
+		Config:     helpers.Clone(li.Config),
+		HostStatus: fly.HostStatusOk,
 	}
 	// also must preserve any user's added metadata except for known fly metadata keys
 	origMachineRaw.Config.Metadata["user-added-me"] = "keep it"
@@ -72,9 +73,10 @@ func Test_launchInputFor_Basic(t *testing.T) {
 
 	// Now updating the machines must include changes to appConfig
 	origMachineRaw = &fly.Machine{
-		ID:     li.ID,
-		Region: li.Region,
-		Config: helpers.Clone(li.Config),
+		ID:         li.ID,
+		Region:     li.Region,
+		Config:     helpers.Clone(li.Config),
+		HostStatus: fly.HostStatusOk,
 	}
 	want.Config.Image = "super/globe"
 	want.Config.Env["NOT_SET_ON_RESTART_ONLY"] = "true"
@@ -83,6 +85,58 @@ func Test_launchInputFor_Basic(t *testing.T) {
 	assert.Equal(t, want, li)
 }
 
+// Test machines on unreachable hosts
+func Test_launchInputForUpdate_HostStatusUnreachable(t *testing.T) {
+	md, err := stabMachineDeployment(&appconfig.Config{
+		AppName:       "my-cool-app",
+		PrimaryRegion: "scl",
+		Env: map[string]string{
+			"OTHER": "value",
+		},
+	})
+	assert.NoError(t, err)
+
+	li, err := md.launchInputForUpdate(&fly.Machine{
+		ID:     "ab1234567890",
+		Region: "ord",
+		Config: &fly.MachineConfig{
+			Metadata: map[string]string{"fly_process_group": "app"},
+		},
+		HostStatus: fly.HostStatusUnreachable,
+	})
+	require.NoError(t, err)
+	require.True(t, li.RequiresReplacement)
+	require.Equal(t, li.Region, "ord")
+
+	// Updating an unreachable machine with a volume attached must fail until we can move the volume to another host
+	md.appConfig.Mounts = []appconfig.Mount{{Source: "data", Destination: "/data"}}
+	li, err = md.launchInputForUpdate(&fly.Machine{
+		ID: "ab1234567890",
+		IncompleteConfig: &fly.MachineConfig{
+			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/data", Name: "data"}},
+		},
+		HostStatus: fly.HostStatusUnreachable,
+	})
+	require.ErrorContains(t, err, "unreachable")
+
+	// Changing the volume name returns a new machine with a different volume attached
+	md.volumes = map[string][]fly.Volume{
+		"data": {
+			{ID: "vol_10001", Name: "data"},
+		},
+	}
+	li, err = md.launchInputForUpdate(&fly.Machine{
+		ID: "ab1234567890",
+		IncompleteConfig: &fly.MachineConfig{
+			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/data", Name: "replace-me-because-i-m-different-fly-toml"}},
+		},
+		HostStatus: fly.HostStatusUnreachable,
+	})
+	require.NoError(t, err)
+	require.True(t, li.RequiresReplacement)
+	require.Equal(t, li.Config.Mounts, []fly.MachineMount{{Path: "/data", Volume: "vol_10001", Name: "data"}})
+}
+
 // Test Mounts
 func Test_launchInputFor_onMounts(t *testing.T) {
 	md, err := stabMachineDeployment(&appconfig.Config{
@@ -109,10 +163,12 @@ func Test_launchInputFor_onMounts(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/data", Name: "data"}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
 	assert.Equal(t, "ab1234567890", li.ID)
+	assert.False(t, li.RequiresReplacement)
 	assert.Equal(t, fly.MachineMount{Volume: "vol_attached", Path: "/data", Name: "data"}, li.Config.Mounts[0])
 
 	// Update a machine with volume attached on a different path
@@ -121,10 +177,12 @@ func Test_launchInputFor_onMounts(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/update-me", Name: "data"}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
 	assert.Equal(t, "ab1234567890", li.ID)
+	assert.False(t, li.RequiresReplacement)
 	assert.Equal(t, fly.MachineMount{Volume: "vol_attached", Path: "/data", Name: "data"}, li.Config.Mounts[0])
 
 	// Updating a machine with an existing unnamed mount must keep the original mount as much as possible
@@ -133,10 +191,12 @@ func Test_launchInputFor_onMounts(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/keep-me"}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
 	assert.Equal(t, "ab1234567890", li.ID)
+	assert.False(t, li.RequiresReplacement)
 	assert.Equal(t, fly.MachineMount{Volume: "vol_attached", Path: "/keep-me"}, li.Config.Mounts[0])
 
 	// Updating a machine whose volume name doesn't match fly.toml's mount section must replace the machine altogether
@@ -145,6 +205,7 @@ func Test_launchInputFor_onMounts(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/replace-me", Name: "replace-me"}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
@@ -159,11 +220,13 @@ func Test_launchInputFor_onMounts(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/replace-me", Name: "replace-me"}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	assert.Equal(t, "ab1234567890", li.ID)
 	assert.True(t, li.RequiresReplacement)
 	assert.Empty(t, li.Config.Mounts)
+
 }
 
 // test mounts with auto volume resize
@@ -213,6 +276,7 @@ func Test_launchInputFor_onMountsAndAutoResize(t *testing.T) {
 				SizeGbLimit:            200,
 			}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
@@ -239,6 +303,7 @@ func Test_launchInputFor_onMountsAndAutoResize(t *testing.T) {
 				SizeGbLimit:            200,
 			}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
@@ -264,6 +329,7 @@ func Test_launchInputFor_onMountsAndAutoResize(t *testing.T) {
 				SizeGbLimit:            200,
 			}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
@@ -282,6 +348,7 @@ func Test_launchInputFor_onMountsAndAutoResize(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{Volume: "vol_attached", Path: "/replace-me", Name: "replace-me"}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, li.Config.Mounts)
@@ -310,6 +377,7 @@ func Test_launchInputFor_onMountsAndAutoResize(t *testing.T) {
 				SizeGbLimit:            200,
 			}},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 	assert.Equal(t, "ab1234567890", li.ID)
@@ -343,6 +411,7 @@ func Test_launchInputForUpdate_keepUnmanagedFields(t *testing.T) {
 				CmdOverride: []string{"foo"},
 			}},
 		},
+		HostStatus: fly.HostStatusOk,
 	}
 	li, err := md.launchInputForUpdate(origMachineRaw)
 	require.NoError(t, err)
@@ -382,6 +451,7 @@ func Test_launchInputForUpdate_clearStandbysWithServices(t *testing.T) {
 		Config: &fly.MachineConfig{
 			Standbys: []string{"xy0987654321"},
 		},
+		HostStatus: fly.HostStatusOk,
 	})
 	require.NoError(t, err)
 
@@ -450,6 +520,7 @@ func Test_launchInputForUpdate_Files(t *testing.T) {
 	require.NoError(t, err)
 
 	li, err := md.launchInputForUpdate(&fly.Machine{
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Files: []*fly.File{
 				{
diff --git a/internal/command/deploy/machines_test.go b/internal/command/deploy/machines_test.go
index c98e9bf84..9ef940d12 100644
--- a/internal/command/deploy/machines_test.go
+++ b/internal/command/deploy/machines_test.go
@@ -182,6 +182,7 @@ func Test_resolveUpdatedMachineConfig_ReleaseCommand(t *testing.T) {
 
 	// Update existing release command machine
 	origMachine := &fly.Machine{
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Env: map[string]string{
 				"PRIMARY_REGION": "different-region",
@@ -268,6 +269,7 @@ func Test_resolveUpdatedMachineConfig_Mounts(t *testing.T) {
 	}, li)
 
 	origMachine := &fly.Machine{
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Mounts: []fly.MachineMount{{
 				Volume: "vol_alreadyattached",
@@ -316,7 +318,8 @@ func Test_resolveUpdatedMachineConfig_restartOnly(t *testing.T) {
 	md.img = "SHOULD-NOT-USE-THIS-TAG"
 
 	origMachine := &fly.Machine{
-		ID: "OrigID",
+		HostStatus: fly.HostStatusOk,
+		ID:         "OrigID",
 		Config: &fly.MachineConfig{
 			Image: "instead-use/the-redmoon",
 		},
@@ -355,7 +358,8 @@ func Test_resolveUpdatedMachineConfig_restartOnlyProcessGroup(t *testing.T) {
 	md.img = "SHOULD-NOT-USE-THIS-TAG"
 
 	origMachine := &fly.Machine{
-		ID: "OrigID",
+		HostStatus: fly.HostStatusOk,
+		ID:         "OrigID",
 		Config: &fly.MachineConfig{
 			Image: "instead-use/the-redmoon",
 			Metadata: map[string]string{
diff --git a/internal/command/deploy/plan_test.go b/internal/command/deploy/plan_test.go
index 1341d119a..10cb12edd 100644
--- a/internal/command/deploy/plan_test.go
+++ b/internal/command/deploy/plan_test.go
@@ -85,7 +85,8 @@ func TestUpdateMachineConfig(t *testing.T) {
 	ctx = iostreams.NewContext(ctx, io)
 
 	oldMachine := &fly.Machine{
-		ID: "machine1",
+		ID:         "machine1",
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Image: "image1",
 			Metadata: map[string]string{
@@ -130,14 +131,16 @@ func TestUpdateMachineConfig(t *testing.T) {
 		},
 		UpdateFunc: func(ctx context.Context, builder fly.LaunchMachineInput, nonce string) (out *fly.Machine, err error) {
 			return &fly.Machine{
-				ID:     builder.ID,
-				Config: builder.Config,
+				ID:         builder.ID,
+				HostStatus: fly.HostStatusOk,
+				Config:     builder.Config,
 			}, nil
 		},
 		LaunchFunc: func(ctx context.Context, builder fly.LaunchMachineInput) (out *fly.Machine, err error) {
 			return &fly.Machine{
-				ID:     builder.ID,
-				Config: builder.Config,
+				ID:         builder.ID,
+				HostStatus: fly.HostStatusOk,
+				Config:     builder.Config,
 			}, nil
 		},
 
@@ -172,31 +175,35 @@ func TestUpdateMachines(t *testing.T) {
 
 	oldMachines := []*fly.Machine{
 		{
-			ID:    "machine1",
-			State: "started",
+			ID:         "machine1",
+			State:      "started",
+			HostStatus: fly.HostStatusOk,
 			Config: &fly.MachineConfig{
 				Image: "image1",
 			},
 		},
 		{
-			ID:    "machine2",
-			State: "started",
+			ID:         "machine2",
+			State:      "started",
+			HostStatus: fly.HostStatusOk,
 			Config: &fly.MachineConfig{
 				Image: "image1",
 			},
 		},
 		{
-			ID:    "machine3",
-			State: "started",
-			Config: &fly.MachineConfig{
+			ID:         "machine3",
+			State:      "started",
+			HostStatus: fly.HostStatusUnreachable,
+			IncompleteConfig: &fly.MachineConfig{
 				Image: "image1",
 			},
 		},
 	}
 	newMachines := lo.Map(oldMachines, func(m *fly.Machine, _ int) *fly.Machine {
 		return &fly.Machine{
-			ID:    m.ID,
-			State: "started",
+			ID:         m.ID,
+			State:      "started",
+			HostStatus: fly.HostStatusOk,
 			Config: &fly.MachineConfig{
 				Image: "image2",
 			},
@@ -213,7 +220,7 @@ func TestUpdateMachines(t *testing.T) {
 			acquiredLeases.Store(machineID, true)
 			return &fly.MachineLease{
 				Data: &fly.MachineLeaseData{
-					Nonce: "nonce",
+					Nonce: machineID + "nonce",
 				},
 			}, nil
 		},
@@ -222,18 +229,23 @@ func TestUpdateMachines(t *testing.T) {
 		},
 		UpdateFunc: func(ctx context.Context, builder fly.LaunchMachineInput, nonce string) (out *fly.Machine, err error) {
 			return &fly.Machine{
-				ID:     builder.ID,
-				Config: builder.Config,
-				State:  "started",
+				ID:         builder.ID,
+				Config:     builder.Config,
+				State:      "started",
+				HostStatus: fly.HostStatusOk,
 			}, nil
 		},
 		LaunchFunc: func(ctx context.Context, builder fly.LaunchMachineInput) (out *fly.Machine, err error) {
 			return &fly.Machine{
-				ID:     builder.ID,
-				Config: builder.Config,
-				State:  "started",
+				ID:         builder.ID,
+				Config:     builder.Config,
+				State:      "started",
+				HostStatus: fly.HostStatusOk,
 			}, nil
 		},
+		DestroyFunc: func(ctx context.Context, input fly.RemoveMachineInput, nonce string) (err error) {
+			return nil
+		},
 		WaitFunc: func(ctx context.Context, machine *fly.Machine, state string, timeout time.Duration) (err error) {
 			if state == "started" {
 				machine.State = "started"
@@ -259,8 +271,9 @@ func TestUpdateMachines(t *testing.T) {
 		},
 		RefreshLeaseFunc: func(ctx context.Context, machineID string, ttl *int, nonce string) (*fly.MachineLease, error) {
 			return &fly.MachineLease{
+				Status: "success",
 				Data: &fly.MachineLeaseData{
-					Nonce: "nonce",
+					Nonce: nonce,
 				},
 			}, nil
 		},
@@ -308,9 +321,10 @@ func TestUpdateMachines(t *testing.T) {
 		}
 
 		return &fly.Machine{
-			ID:     builder.ID,
-			Config: builder.Config,
-			State:  "started",
+			ID:         builder.ID,
+			Config:     builder.Config,
+			State:      "started",
+			HostStatus: fly.HostStatusOk,
 		}, nil
 	}
 	acquiredLeases = sync.Map{}
@@ -332,9 +346,10 @@ func TestUpdateMachines(t *testing.T) {
 			return nil, &unrecoverableError{err: assert.AnError}
 		} else {
 			return &fly.Machine{
-				ID:     builder.ID,
-				Config: builder.Config,
-				State:  "started",
+				ID:         builder.ID,
+				Config:     builder.Config,
+				State:      "started",
+				HostStatus: fly.HostStatusOk,
 			}, nil
 		}
 	}
@@ -380,6 +395,7 @@ func TestUpdateOrCreateMachine(t *testing.T) {
 				ID:         builder.ID,
 				Config:     builder.Config,
 				State:      "started",
+				HostStatus: fly.HostStatusOk,
 				LeaseNonce: "nonce",
 			}, nil
 		},
@@ -389,20 +405,23 @@ func TestUpdateOrCreateMachine(t *testing.T) {
 				ID:         builder.ID,
 				Config:     builder.Config,
 				State:      "started",
+				HostStatus: fly.HostStatusOk,
 				LeaseNonce: "nonce",
 			}, nil
 		},
 	}
 
 	oldMachine := &fly.Machine{
-		ID: "machine1",
+		ID:         "machine1",
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Image: "image1",
 		},
 		LeaseNonce: "nonce",
 	}
 	newMachine := &fly.Machine{
-		ID: "machine1",
+		ID:         "machine1",
+		HostStatus: fly.HostStatusOk,
 		Config: &fly.MachineConfig{
 			Image: "image2",
 		},