static int doDetachVolume(struct nc_state_t *nc, ncMetadata * pMeta, char *instanceId, char *volumeId, char *attachmentToken, char *localDev, int force, int grab_inst_sem)
{
    int ret = EUCA_OK;
    int have_remote_device = 0;
    char *xml = NULL;
    char resourceName[1][MAX_SENSOR_NAME_LEN] = { {0} };
    char resourceAlias[1][MAX_SENSOR_NAME_LEN] = { {0} };
    char volpath[MAX_PATH];
    char lvolpath[MAX_PATH];
    char *tagBuf;
    char *localDevName;
    char localDevReal[32], localDevTag[256], remoteDevReal[132];
    char scUrl[512];
    char *connectionString = NULL;
    instance_states lastState = NO_STATE;
    char *remoteDevStr = NULL;
    ncVolume *volume = NULL;

    if (!strcmp(nc->H->name, "xen")) {
        tagBuf = NULL;
        localDevName = localDevReal;
    } else if (!strcmp(nc->H->name, "kvm")) {
        tagBuf = localDevTag;
        localDevName = localDevTag;
    } else {
        LOGERROR("[%s][%s] unknown hypervisor type '%s'\n", instanceId, volumeId, nc->H->name);
        return EUCA_ERROR;
    }

    // get the file name from the device path and, for KVM, the "unknown" string
    ret = convert_dev_names(localDev, localDevReal, tagBuf);
    if (ret)
        return ret;

    {                                  // find the instance record and work with it
        if (grab_inst_sem)
            sem_p(inst_sem);

        ncInstance *instance = find_instance(&global_instances, instanceId);
        if (instance == NULL) {
            LOGERROR("[%s][%s] failed to find instance for volume detachment\n", instanceId, volumeId);
            if (grab_inst_sem)
                sem_v(inst_sem);
            return EUCA_NOT_FOUND_ERROR;
        }
        // set up paths for later
        snprintf(volpath, sizeof(volpath), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volumeId);    // vol-XXX.xml
        snprintf(lvolpath, sizeof(lvolpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volumeId);  // vol-XXX-libvirt.xml

        // save current state on the stack
        lastState = instance->state;

        // mark volume as 'detaching', do not over-write the attachment token used for
        volume = save_volume(instance, volumeId, NULL, NULL, localDevName, localDevReal, VOL_STATE_DETACHING);
        if (!volume) {
            LOGERROR("[%s][%s] failed to update the volume record, aborting volume detachment\n", instanceId, volumeId);
            if (grab_inst_sem)
                sem_v(inst_sem);
            return EUCA_ERROR;
        }
        save_instance_struct(instance);
        copy_instances();

        // lookup the volume info locally for detachment
        if (volume->connectionString[0] == '\0' || volume->attachmentToken == NULL || volume->attachmentToken[0] == '\0') {
            LOGERROR("[%s][%s] failed to find the local volume attachment record, aborting volume detachment\n", instanceId, volumeId);
            if (grab_inst_sem)
                sem_v(inst_sem);
            return EUCA_ERROR;
        }
        // do iscsi connect shellout if remoteDev is an iSCSI target
        // get credentials, decrypt them
        // (used to have check if iscsi here, not necessary with AOE deprecation.)
        remoteDevStr = get_volume_local_device(volume->connectionString);
        if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
            LOGERROR("[%s][%s] failed to get local name of host iscsi device\n", instanceId, volumeId);
            remoteDevReal[0] = '\0';
        } else {
            snprintf(remoteDevReal, sizeof(remoteDevReal), "%s", remoteDevStr);
            have_remote_device = 1;
        }
        EUCA_FREE(remoteDevStr);

        if (grab_inst_sem)
            sem_v(inst_sem);
    }

    // something went wrong above, abort
    if (!have_remote_device) {
        ret = EUCA_ERROR;
        goto disconnect;
    }
    // make sure there is a block device
    if (check_block(remoteDevReal)) {
        LOGERROR("[%s][%s] cannot verify that host device '%s' is available for hypervisor detach\n", instanceId, volumeId, remoteDevReal);
        if (!force)
            ret = EUCA_ERROR;
        goto disconnect;
    }
    // read in libvirt XML
    xml = file2str(lvolpath);
    if (xml == NULL) {
        LOGERROR("[%s][%s] failed to read volume XML from %s\n", instanceId, volumeId, lvolpath);
        ret = EUCA_ERROR;
        goto disconnect;
    }

    {                                  // connect to hypervisor and do
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            ret = EUCA_HYPERVISOR_ERROR;
            goto disconnect;
        }
        // refresh stats so volume measurements are captured before it disappears
        euca_strncpy(resourceName[0], instanceId, MAX_SENSOR_NAME_LEN);
        sensor_refresh_resources(resourceName, resourceAlias, 1);

        // find domain on hypervisor
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        if (dom == NULL) {
            if (lastState != BOOTING && lastState != STAGING) {
                LOGWARN("[%s][%s] domain not running on hypervisor, will not try to detach device\n", instanceId, volumeId);
            }
            unlock_hypervisor_conn();
            goto disconnect;
        }
        // protect libvirt calls because we've seen problems during concurrent libvirt invocation
        int err = virDomainDetachDevice(dom, xml);
        if (!strcmp(nc->H->name, "xen")) {
            err = xen_detach_helper(nc, instanceId, localDevReal, xml);
        }

        if (err) {
            LOGERROR("[%s][%s] failed to detach host device '%s' from guest device '%s'\n", instanceId, volumeId, remoteDevReal, localDevReal);
            LOGERROR("[%s][%s] virDomainDetachDevice() or 'virsh detach' failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
            if (!force)
                ret = EUCA_HYPERVISOR_ERROR;
        } else {
            call_hooks(NC_EVENT_POST_DETACH, volpath);  // invoke hooks, but do not do anything if they return error
            unlink(lvolpath);          // remove vol-XXX-libvirt.xml
            unlink(volpath);           // remove vol-XXXX.xml file
        }

        virDomainFree(dom);            // release libvirt resource
        unlock_hypervisor_conn();      // unlock the connection to the hypervisor
    }

disconnect:

    {                                  // update the instance structure while under a lock
        if (grab_inst_sem)
            sem_p(inst_sem);

        ncInstance *instance = find_instance(&global_instances, instanceId);
        if (instance == NULL) {
            LOGWARN("[%s][%s] failed to find the instance to update volume state\n", instanceId, volumeId);
        } else {
            // record volume state in memory and on disk
            char *next_vol_state;
            if (ret == EUCA_OK) {
                next_vol_state = VOL_STATE_DETACHED;
            } else {
                next_vol_state = VOL_STATE_DETACHING_FAILED;
            }
            ncVolume *volume = save_volume(instance, volumeId, NULL, NULL, NULL, NULL, next_vol_state);
            if (volume == NULL) {
                LOGWARN("[%s][%s] failed to save the volume record\n", instanceId, volumeId);
                ret = EUCA_ERROR;
            } else {
                connectionString = strdup(volume->connectionString);
            }
            save_instance_struct(instance);
            copy_instances();
            update_disk_aliases(instance);  // ask sensor subsystem to stop tracking the volume
        }

        if (grab_inst_sem)
            sem_v(inst_sem);
    }

    // if iSCSI, try to disconnect the target
    if (have_remote_device) {
        //Do the ebs disconnect.
        LOGTRACE("[%s][%s] Disconnecting EBS volume to local host\n", instanceId, volumeId);
        if (get_service_url("storage", nc, scUrl) != EUCA_OK || strlen(scUrl) == 0) {
            LOGERROR("Could not find SC URL for making unexport call.\n");
            ret = EUCA_ERROR;
        } else {
            LOGTRACE("[%s][%s] Using SC Url: %s\n", instanceId, volumeId, scUrl);
            //Use the volume attachment token from the initial attachment instead of the one that came over the wire. This ensures parity between attach/detach.
            if (disconnect_ebs_volume(scUrl, nc->config_use_ws_sec, nc->config_sc_policy_file, volume->attachmentToken, connectionString, nc->ip, nc->iqn) != EUCA_OK) {
                LOGERROR("[%s][%s] failed to disconnect iscsi target\n", instanceId, volumeId);
                if (!force)
                    ret = EUCA_ERROR;

            }
        }
    }

    if (ret == EUCA_OK)
        LOGINFO("[%s][%s] detached as host device '%s' and guest device '%s'\n", instanceId, volumeId, remoteDevReal, localDevReal);
    // remoteDev can be a long string, so to keep log readable, we log it at TRACE level unless there was a problem
    int log_level_for_devstring = EUCA_LOG_TRACE;
    if (ret != EUCA_OK)
        log_level_for_devstring = EUCA_LOG_DEBUG;
    EUCALOG(log_level_for_devstring, "[%s][%s] remote device string: %s\n", instanceId, volumeId, connectionString);
    EUCA_FREE(connectionString);
    EUCA_FREE(xml);

    if (force) {
        return (EUCA_OK);
    }
    return ret;
}
static int doDetachVolume(struct nc_state_t *nc, ncMetadata * pMeta, char *instanceId, char *volumeId, char *attachmentToken, char *localDev, int force, int grab_inst_sem)
{
    int ret = EUCA_OK;
    int have_remote_device = 0;
    char *xml = NULL;
    char resourceName[1][MAX_SENSOR_NAME_LEN] = { {0} };
    char resourceAlias[1][MAX_SENSOR_NAME_LEN] = { {0} };
    char volpath[MAX_PATH];
    char lvolpath[MAX_PATH];
    char *tagBuf;
    char *localDevName;
    char localDevReal[32], localDevTag[256], remoteDevReal[132];
    char scUrl[512];
    char *connectionString = NULL;
    instance_states lastState = NO_STATE;
    char *remoteDevStr = NULL;
    ncVolume *volume = NULL;

    if (!strcmp(nc->H->name, "xen")) {
        tagBuf = NULL;
        localDevName = localDevReal;
    } else if (!strcmp(nc->H->name, "kvm")) {
        tagBuf = localDevTag;
        localDevName = localDevTag;
    } else {
        LOGERROR("[%s][%s] unknown hypervisor type '%s'\n", instanceId, volumeId, nc->H->name);
        return EUCA_ERROR;
    }

     
    ret = convert_dev_names(localDev, localDevReal, tagBuf);
    if (ret)
        return ret;

    {                                   
        if (grab_inst_sem)
            sem_p(inst_sem);

        ncInstance *instance = find_instance(&global_instances, instanceId);
        if (instance == NULL) {
            LOGERROR("[%s][%s] failed to find instance for volume detachment\n", instanceId, volumeId);
            if (grab_inst_sem)
                sem_v(inst_sem);
            return EUCA_NOT_FOUND_ERROR;
        }
         
        snprintf(volpath, sizeof(volpath), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volumeId);     
        snprintf(lvolpath, sizeof(lvolpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volumeId);   

         
        lastState = instance->state;

         
        volume = save_volume(instance, volumeId, NULL, NULL, localDevName, localDevReal, VOL_STATE_DETACHING);
        if (!volume) {
            LOGERROR("[%s][%s] failed to update the volume record, aborting volume detachment\n", instanceId, volumeId);
            if (grab_inst_sem)
                sem_v(inst_sem);
            return EUCA_ERROR;
        }
        save_instance_struct(instance);
        copy_instances();

         
        if (volume->connectionString[0] == '\0' || volume->attachmentToken == NULL || volume->attachmentToken[0] == '\0') {
            LOGERROR("[%s][%s] failed to find the local volume attachment record, aborting volume detachment\n", instanceId, volumeId);
            if (grab_inst_sem)
                sem_v(inst_sem);
            return EUCA_ERROR;
        }
         
         
         
        remoteDevStr = get_volume_local_device(volume->connectionString);
        if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
            LOGERROR("[%s][%s] failed to get local name of host iscsi device\n", instanceId, volumeId);
            remoteDevReal[0] = '\0';
        } else {
            snprintf(remoteDevReal, sizeof(remoteDevReal), "%s", remoteDevStr);
            have_remote_device = 1;
        }
        EUCA_FREE(remoteDevStr);

        if (grab_inst_sem)
            sem_v(inst_sem);
    }

     
    if (!have_remote_device) {
        ret = EUCA_ERROR;
        goto disconnect;
    }
     
    if (check_block(remoteDevReal)) {
        LOGERROR("[%s][%s] cannot verify that host device '%s' is available for hypervisor detach\n", instanceId, volumeId, remoteDevReal);
        if (!force)
            ret = EUCA_ERROR;
        goto disconnect;
    }
     
    xml = file2str(lvolpath);
    if (xml == NULL) {
        LOGERROR("[%s][%s] failed to read volume XML from %s\n", instanceId, volumeId, lvolpath);
        ret = EUCA_ERROR;
        goto disconnect;
    }

    {                                   
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            ret = EUCA_HYPERVISOR_ERROR;
            goto disconnect;
        }
         
        euca_strncpy(resourceName[0], instanceId, MAX_SENSOR_NAME_LEN);
        sensor_refresh_resources(resourceName, resourceAlias, 1);

         
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        if (dom == NULL) {
            if (lastState != BOOTING && lastState != STAGING) {
                LOGWARN("[%s][%s] domain not running on hypervisor, will not try to detach device\n", instanceId, volumeId);
            }
            unlock_hypervisor_conn();
            goto disconnect;
        }
         
        int err = virDomainDetachDevice(dom, xml);
        if (!strcmp(nc->H->name, "xen")) {
            err = xen_detach_helper(nc, instanceId, localDevReal, xml);
        }

        if (err) {
            LOGERROR("[%s][%s] failed to detach host device '%s' from guest device '%s'\n", instanceId, volumeId, remoteDevReal, localDevReal);
            LOGERROR("[%s][%s] virDomainDetachDevice() or 'virsh detach' failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
            if (!force)
                ret = EUCA_HYPERVISOR_ERROR;
        } else {
            call_hooks(NC_EVENT_POST_DETACH, volpath);   
            unlink(lvolpath);           
            unlink(volpath);            
        }

        virDomainFree(dom);             
        unlock_hypervisor_conn();       
    }

disconnect:

    {                                   
        if (grab_inst_sem)
            sem_p(inst_sem);

        ncInstance *instance = find_instance(&global_instances, instanceId);
        if (instance == NULL) {
            LOGWARN("[%s][%s] failed to find the instance to update volume state\n", instanceId, volumeId);
        } else {
             
            char *next_vol_state;
            if (ret == EUCA_OK) {
                next_vol_state = VOL_STATE_DETACHED;
            } else {
                next_vol_state = VOL_STATE_DETACHING_FAILED;
            }
            ncVolume *volume = save_volume(instance, volumeId, NULL, NULL, NULL, NULL, next_vol_state);
            if (volume == NULL) {
                LOGWARN("[%s][%s] failed to save the volume record\n", instanceId, volumeId);
                ret = EUCA_ERROR;
            } else {
                connectionString = strdup(volume->connectionString);
            }
            save_instance_struct(instance);
            copy_instances();
            update_disk_aliases(instance);   
        }

        if (grab_inst_sem)
            sem_v(inst_sem);
    }

     
    if (have_remote_device) {
         
        LOGTRACE("[%s][%s] Disconnecting EBS volume to local host\n", instanceId, volumeId);
        if (get_service_url("storage", nc, scUrl) != EUCA_OK || strlen(scUrl) == 0) {
            LOGERROR("Could not find SC URL for making unexport call.\n");
            ret = EUCA_ERROR;
        } else {
            LOGTRACE("[%s][%s] Using SC Url: %s\n", instanceId, volumeId, scUrl);
             
            if (disconnect_ebs_volume(scUrl, nc->config_use_ws_sec, nc->config_sc_policy_file, volume->attachmentToken, connectionString, nc->ip, nc->iqn) != EUCA_OK) {
                LOGERROR("[%s][%s] failed to disconnect iscsi target\n", instanceId, volumeId);
                if (!force)
                    ret = EUCA_ERROR;

            }
        }
    }

    if (ret == EUCA_OK)
        LOGINFO("[%s][%s] detached as host device '%s' and guest device '%s'\n", instanceId, volumeId, remoteDevReal, localDevReal);
     
    int log_level_for_devstring = EUCA_LOG_TRACE;
    if (ret != EUCA_OK)
        log_level_for_devstring = EUCA_LOG_DEBUG;
    EUCALOG(log_level_for_devstring, "[%s][%s] remote device string: %s\n", instanceId, volumeId, connectionString);
    EUCA_FREE(connectionString);
    EUCA_FREE(xml);

    if (force) {
        return (EUCA_OK);
    }
    return ret;
}