func (os *OpenStack) DetachDisk(instanceID, volumeID string) error {
	volume, err := os.getVolume(volumeID)
	if err != nil {
		return err
	}
	if volume.Status == volumeAvailableStatus {
		klog.V(2).Infof("volume: %s has been detached from compute: %s ", volume.ID, instanceID)
		return nil
	}
	if volume.Status != volumeInUseStatus {
		return fmt.Errorf("can not detach volume %s, its status is %s", volume.Name, volume.Status)
	}
	cClient, err := os.NewComputeV2()
	if err != nil {
		return err
	}
	if volume.AttachedServerID != instanceID {
		return fmt.Errorf("disk: %s has no attachments or is not attached to compute: %s", volume.Name, instanceID)
	}
	startTime := time.Now()
	err = volumeattach.Delete(cClient, instanceID, volume.ID).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("detach_disk", timeTaken, err)
	if err != nil {
		return fmt.Errorf("failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err)
	}
	klog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID)
	return nil
}
DetachDisk detaches given cinder volume from the compute running kubelet
package openstack
import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"
	"time"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/types"
	cloudprovider "k8s.io/cloud-provider"
	cloudvolume "k8s.io/cloud-provider/volume"
	volerr "k8s.io/cloud-provider/volume/errors"
	volumehelpers "k8s.io/cloud-provider/volume/helpers"
	"k8s.io/component-base/metrics"
	"github.com/gophercloud/gophercloud"
	volumeexpand "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions"
	volumes_v1 "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes"
	volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
	volumes_v3 "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"k8s.io/klog/v2"
)
type volumeService interface {
	createVolume(opts volumeCreateOpts) (string, string, error)
	getVolume(volumeID string) (Volume, error)
	deleteVolume(volumeName string) error
	expandVolume(volumeID string, newSize int) error
}
type VolumesV1 struct {
	blockstorage *gophercloud.ServiceClient
	opts         BlockStorageOpts
}
type VolumesV2 struct {
	blockstorage *gophercloud.ServiceClient
	opts         BlockStorageOpts
}
type VolumesV3 struct {
	blockstorage *gophercloud.ServiceClient
	opts         BlockStorageOpts
}
type Volume struct {
	AttachedServerID string
	AttachedDevice string
	AvailabilityZone string
	ID string
	Name string
	Status string
	Size int
}
type volumeCreateOpts struct {
	Size         int
	Availability string
	Name         string
	VolumeType   string
	Metadata     map[string]string
}
var _ cloudprovider.PVLabeler = (*OpenStack)(nil)
const (
	volumeAvailableStatus = "available"
	volumeInUseStatus     = "in-use"
	volumeDeletedStatus   = "deleted"
	volumeErrorStatus     = "error"
	newtonMetadataVersion = "2016-06-30"
)
func (volumes *VolumesV1) createVolume(opts volumeCreateOpts) (string, string, error) {
	startTime := time.Now()
	createOpts := volumes_v1.CreateOpts{
		Name:             opts.Name,
		Size:             opts.Size,
		VolumeType:       opts.VolumeType,
		AvailabilityZone: opts.Availability,
		Metadata:         opts.Metadata,
	}
	vol, err := volumes_v1.Create(volumes.blockstorage, createOpts).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("create_v1_volume", timeTaken, err)
	if err != nil {
		return "", "", err
	}
	return vol.ID, vol.AvailabilityZone, nil
}
func (volumes *VolumesV2) createVolume(opts volumeCreateOpts) (string, string, error) {
	startTime := time.Now()
	createOpts := volumes_v2.CreateOpts{
		Name:             opts.Name,
		Size:             opts.Size,
		VolumeType:       opts.VolumeType,
		AvailabilityZone: opts.Availability,
		Metadata:         opts.Metadata,
	}
	vol, err := volumes_v2.Create(volumes.blockstorage, createOpts).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("create_v2_volume", timeTaken, err)
	if err != nil {
		return "", "", err
	}
	return vol.ID, vol.AvailabilityZone, nil
}
func (volumes *VolumesV3) createVolume(opts volumeCreateOpts) (string, string, error) {
	startTime := time.Now()
	createOpts := volumes_v3.CreateOpts{
		Name:             opts.Name,
		Size:             opts.Size,
		VolumeType:       opts.VolumeType,
		AvailabilityZone: opts.Availability,
		Metadata:         opts.Metadata,
	}
	vol, err := volumes_v3.Create(volumes.blockstorage, createOpts).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("create_v3_volume", timeTaken, err)
	if err != nil {
		return "", "", err
	}
	return vol.ID, vol.AvailabilityZone, nil
}
func (volumes *VolumesV1) getVolume(volumeID string) (Volume, error) {
	startTime := time.Now()
	volumeV1, err := volumes_v1.Get(volumes.blockstorage, volumeID).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("get_v1_volume", timeTaken, err)
	if err != nil {
		if isNotFound(err) {
			return Volume{}, ErrNotFound
		}
		return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
	}
	volume := Volume{
		AvailabilityZone: volumeV1.AvailabilityZone,
		ID:               volumeV1.ID,
		Name:             volumeV1.Name,
		Status:           volumeV1.Status,
		Size:             volumeV1.Size,
	}
	if len(volumeV1.Attachments) > 0 && volumeV1.Attachments[0]["server_id"] != nil {
		volume.AttachedServerID = volumeV1.Attachments[0]["server_id"].(string)
		volume.AttachedDevice = volumeV1.Attachments[0]["device"].(string)
	}
	return volume, nil
}
func (volumes *VolumesV2) getVolume(volumeID string) (Volume, error) {
	startTime := time.Now()
	volumeV2, err := volumes_v2.Get(volumes.blockstorage, volumeID).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("get_v2_volume", timeTaken, err)
	if err != nil {
		if isNotFound(err) {
			return Volume{}, ErrNotFound
		}
		return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
	}
	volume := Volume{
		AvailabilityZone: volumeV2.AvailabilityZone,
		ID:               volumeV2.ID,
		Name:             volumeV2.Name,
		Status:           volumeV2.Status,
		Size:             volumeV2.Size,
	}
	if len(volumeV2.Attachments) > 0 {
		volume.AttachedServerID = volumeV2.Attachments[0].ServerID
		volume.AttachedDevice = volumeV2.Attachments[0].Device
	}
	return volume, nil
}
func (volumes *VolumesV3) getVolume(volumeID string) (Volume, error) {
	startTime := time.Now()
	volumeV3, err := volumes_v3.Get(volumes.blockstorage, volumeID).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("get_v3_volume", timeTaken, err)
	if err != nil {
		if isNotFound(err) {
			return Volume{}, ErrNotFound
		}
		return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
	}
	volume := Volume{
		AvailabilityZone: volumeV3.AvailabilityZone,
		ID:               volumeV3.ID,
		Name:             volumeV3.Name,
		Status:           volumeV3.Status,
		Size:             volumeV3.Size,
	}
	if len(volumeV3.Attachments) > 0 {
		volume.AttachedServerID = volumeV3.Attachments[0].ServerID
		volume.AttachedDevice = volumeV3.Attachments[0].Device
	}
	return volume, nil
}
func (volumes *VolumesV1) deleteVolume(volumeID string) error {
	startTime := time.Now()
	err := volumes_v1.Delete(volumes.blockstorage, volumeID).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("delete_v1_volume", timeTaken, err)
	return err
}
func (volumes *VolumesV2) deleteVolume(volumeID string) error {
	startTime := time.Now()
	err := volumes_v2.Delete(volumes.blockstorage, volumeID, nil).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("delete_v2_volume", timeTaken, err)
	return err
}
func (volumes *VolumesV3) deleteVolume(volumeID string) error {
	startTime := time.Now()
	err := volumes_v3.Delete(volumes.blockstorage, volumeID, nil).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("delete_v3_volume", timeTaken, err)
	return err
}
func (volumes *VolumesV1) expandVolume(volumeID string, newSize int) error {
	startTime := time.Now()
	createOpts := volumeexpand.ExtendSizeOpts{
		NewSize: newSize,
	}
	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, createOpts).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
	return err
}
func (volumes *VolumesV2) expandVolume(volumeID string, newSize int) error {
	startTime := time.Now()
	createOpts := volumeexpand.ExtendSizeOpts{
		NewSize: newSize,
	}
	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, createOpts).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
	return err
}
func (volumes *VolumesV3) expandVolume(volumeID string, newSize int) error {
	startTime := time.Now()
	createOpts := volumeexpand.ExtendSizeOpts{
		NewSize: newSize,
	}
	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, createOpts).ExtractErr()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
	return err
}
func (os *OpenStack) OperationPending(diskName string) (bool, string, error) {
	volume, err := os.getVolume(diskName)
	if err != nil {
		return false, "", err
	}
	volumeStatus := volume.Status
	if volumeStatus == volumeErrorStatus {
		err = fmt.Errorf("status of volume %s is %s", diskName, volumeStatus)
		return false, volumeStatus, err
	}
	if volumeStatus == volumeAvailableStatus || volumeStatus == volumeInUseStatus || volumeStatus == volumeDeletedStatus {
		return false, volume.Status, nil
	}
	return true, volumeStatus, nil
}
func (os *OpenStack) AttachDisk(instanceID, volumeID string) (string, error) {
	volume, err := os.getVolume(volumeID)
	if err != nil {
		return "", err
	}
	cClient, err := os.NewComputeV2()
	if err != nil {
		return "", err
	}
	if volume.AttachedServerID != "" {
		if instanceID == volume.AttachedServerID {
			klog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID)
			return volume.ID, nil
		}
		nodeName, err := os.GetNodeNameByID(volume.AttachedServerID)
		attachErr := fmt.Sprintf("disk %s path %s is attached to a different instance (%s)", volumeID, volume.AttachedDevice, volume.AttachedServerID)
		if err != nil {
			klog.Error(attachErr)
			return "", errors.New(attachErr)
		}
		devicePath := volume.AttachedDevice
		danglingErr := volerr.NewDanglingError(attachErr, nodeName, devicePath)
		klog.V(2).Infof("Found dangling volume %s attached to node %s", volumeID, nodeName)
		return "", danglingErr
	}
	startTime := time.Now()
	_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
		VolumeID: volume.ID,
	}).Extract()
	timeTaken := time.Since(startTime).Seconds()
	recordOpenstackOperationMetric("attach_disk", timeTaken, err)
	if err != nil {
		return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err)
	}
	klog.V(2).Infof("Successfully attached %s volume to %s compute", volumeID, instanceID)
	return volume.ID, nil
}