Repository: linode/linode-cloud-controller-manager (go)

Issue:
CCM does not properly detect nodes that are powered off
## Bug Reporting
CCM does not properly detect nodes that are powered off.

### Expected Behavior
On shutdown of a Kubernetes node, the CCM detects that it is powered off and migrates workloads off of the node.

### Actual Behavior
The node status becomes `NotReady` after a few minutes because kubelet stops responding, but pods still have a status of `Running` and do not get rescheduled.

### Steps to Reproduce the Problem
  1. Shut down a Kubernetes node on a cluster running the CCM, wait 5 minutes
  2. Observe node status with `kubectl get nodes`, note that the down node has a status of `NotReady`
  3. Observe pod status with `kubectl get pods -A`, note that pods on the node are not rescheduled


Pull request description:
Add support to detect for powered-off nodes
Should close #67 

When a node becomes `NotReady`, the CCM's node controller will call upon `InstanceShutdownByProviderID` to determine whether or not the node has been powered off. If the cloud provider has implemented this function, it will return true when the instance is deemed to be shutdown, and the CCM will then proceed to add [shutdown-based taints](https://github.com/kubernetes/kubernetes/blob/cb303e613a121a29364f75cc67d3d580833a7479/staging/src/k8s.io/cloud-provider/controllers/nodelifecycle/node_lifecycle_controller.go#L181) that will evict the pods from the node.

### General:

* [x] Have you removed all sensitive information, including but not limited to access keys and passwords?
* [x] Have you checked to ensure there aren't other open or closed [Pull Requests](../../pulls) for the same bug/feature/question?

### Pull Request Guidelines:

1. [x] Does your submission pass tests?
1. [x] Have you added tests? 
1. [x] Are you addressing a single feature in this PR? 
1. [x] Are your commits atomic, addressing one change per commit?
1. [x] Are you following the conventions of the language? 
1. [ ] Have you saved your large formatting changes for a different PR, so we can focus on your work?
1. [x] Have you explained your rationale for why this feature is needed? 
1. [x] Have you linked your PR to an [open issue](https://blog.github.com/2013-05-14-closing-issues-via-pull-requests/)



Patch:
diff --git a/cloud/linode/instances.go b/cloud/linode/instances.go
index e5199b8..f39a11c 100644
--- a/cloud/linode/instances.go
+++ b/cloud/linode/instances.go
@@ -176,5 +176,29 @@ func (i *instances) InstanceExistsByProviderID(ctx context.Context, providerID s
 }
 
 func (i *instances) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
-	return false, cloudprovider.NotImplemented
+	ctx = sentry.SetHubOnContext(ctx)
+	sentry.SetTag(ctx, "provider_id", providerID)
+
+	id, err := parseProviderID(providerID)
+	if err != nil {
+		sentry.CaptureError(ctx, err)
+		return false, err
+	}
+
+	sentry.SetTag(ctx, "linode_id", strconv.Itoa(id))
+
+	instance, err := linodeByID(ctx, i.client, id)
+	if err != nil {
+		sentry.CaptureError(ctx, err)
+		return false, err
+	}
+
+	// An instance is considered to be "shutdown" when it is
+	// in the process of shutting down, or already offline.
+	if instance.Status == linodego.InstanceOffline ||
+		instance.Status == linodego.InstanceShuttingDown {
+		return true, nil
+	}
+
+	return false, nil
 }


Test patch:
diff --git a/cloud/linode/instances_test.go b/cloud/linode/instances_test.go
index e2c282e..7b9cbd7 100644
--- a/cloud/linode/instances_test.go
+++ b/cloud/linode/instances_test.go
@@ -314,7 +314,56 @@ func TestAddSSHKeyToAllInstances(t *testing.T) {
 }
 
 func TestInstanceShutdownByProviderID(t *testing.T) {
-	instances := newInstances(nil)
-	_, err := instances.InstanceShutdownByProviderID(context.TODO(), providerIDPrefix+"12345")
-	assert.ErrorIs(t, err, cloudprovider.NotImplemented)
+	ctx := context.TODO()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	client := NewMockClient(ctrl)
+	instances := newInstances(client)
+
+	t.Run("fails when instance not found", func(t *testing.T) {
+		id := 12345
+		providerID := providerIDPrefix + strconv.Itoa(id)
+		client.EXPECT().GetInstance(gomock.Any(), id).Times(1).Return(nil, linodego.Error{Code: http.StatusNotFound})
+		shutdown, err := instances.InstanceShutdownByProviderID(ctx, providerID)
+
+		assert.Error(t, err)
+		assert.False(t, shutdown)
+	})
+
+	t.Run("returns true when instance is shut down", func(t *testing.T) {
+		id := 12345
+		providerID := providerIDPrefix + strconv.Itoa(id)
+		client.EXPECT().GetInstance(gomock.Any(), id).Times(1).Return(&linodego.Instance{
+			ID: id, Label: "offline-linode", Status: linodego.InstanceOffline,
+		}, nil)
+		shutdown, err := instances.InstanceShutdownByProviderID(ctx, providerID)
+
+		assert.NoError(t, err)
+		assert.True(t, shutdown)
+	})
+
+	t.Run("returns true when instance is shutting down", func(t *testing.T) {
+		id := 12345
+		providerID := providerIDPrefix + strconv.Itoa(id)
+		client.EXPECT().GetInstance(gomock.Any(), id).Times(1).Return(&linodego.Instance{
+			ID: id, Label: "shutting-down-linode", Status: linodego.InstanceShuttingDown,
+		}, nil)
+		shutdown, err := instances.InstanceShutdownByProviderID(ctx, providerID)
+
+		assert.NoError(t, err)
+		assert.True(t, shutdown)
+	})
+
+	t.Run("returns false when instance is running", func(t *testing.T) {
+		id := 12345
+		providerID := providerIDPrefix + strconv.Itoa(id)
+		client.EXPECT().GetInstance(gomock.Any(), id).Times(1).Return(&linodego.Instance{
+			ID: id, Label: "running-linode", Status: linodego.InstanceRunning,
+		}, nil)
+		shutdown, err := instances.InstanceShutdownByProviderID(ctx, providerID)
+
+		assert.NoError(t, err)
+		assert.False(t, shutdown)
+	})
 }