static int doAttachVolume(struct nc_state_t *nc, ncMetadata * pMeta, char *instanceId, char *volumeId, char *attachmentToken, char *localDev)
{
    int ret = EUCA_OK;
    int have_remote_device = 0;
    char *xml = NULL;
    char *tagBuf;
    char *localDevName;
    char localDevReal[32], localDevTag[256], remoteDevReal[132];
    char scUrl[512];                   //Tmp holder for sc url for sc call.
    ncVolume *volume;
    ebs_volume_data *vol_data = NULL;
    char *remoteDevStr = NULL;

    if (!strcmp(nc->H->name, "xen")) {
        tagBuf = NULL;
        localDevName = localDevReal;
    } else if (!strcmp(nc->H->name, "kvm")) {
        tagBuf = localDevTag;
        localDevName = localDevTag;
    } else {
        LOGERROR("[%s][%s] unknown hypervisor type '%s'\n", instanceId, volumeId, nc->H->name);
        return EUCA_ERROR;
    }

    // sets localDevReal to the file name from the device path
    // and, for KVM, sets localDevTag to the "unknown" string
    ret = convert_dev_names(localDev, localDevReal, tagBuf);
    if (ret)
        return ret;

    // find the instance record
    sem_p(inst_sem);
    ncInstance *instance = find_instance(&global_instances, instanceId);
    sem_v(inst_sem);
    if (instance == NULL) {
        return EUCA_NOT_FOUND_ERROR;
    }

    {                                  // connect to hypervisor and query it
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            return EUCA_HYPERVISOR_ERROR;
        }
        // find domain on hypervisor
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        if (dom == NULL) {
            if (instance->state != BOOTING && instance->state != STAGING) {
                LOGWARN("[%s][%s] domain not running on hypervisor, cannot attach device\n", instanceId, volumeId);
            }
            unlock_hypervisor_conn();
            return EUCA_HYPERVISOR_ERROR;
        }
        virDomainFree(dom);            // release libvirt resource
        unlock_hypervisor_conn();
    }

    // mark volume as 'attaching', save token value as we got from the wire
    sem_p(inst_sem);
    {
        volume = save_volume(instance, volumeId, attachmentToken, NULL, localDevName, localDevReal, VOL_STATE_ATTACHING);
        save_instance_struct(instance);
        copy_instances();
    }
    sem_v(inst_sem);

    if (!volume) {
        LOGERROR("[%s][%s] failed to update the volume record, aborting volume attachment\n", instanceId, volumeId);
        return EUCA_ERROR;
    }
    //Do the ebs connect.
    LOGTRACE("[%s][%s] Connecting EBS volume to local host\n", instanceId, volumeId);
    if (get_service_url("storage", nc, scUrl) != EUCA_OK || strlen(scUrl) == 0) {
        LOGERROR("[%s][%s] Failed to lookup enabled Storage Controller. Cannot attach volume: %s\n", instanceId, volumeId, volumeId);
        have_remote_device = 0;
        ret = EUCA_ERROR;
        goto release;
    } else {
        LOGTRACE("[%s][%s] Using SC URL: %s\n", instanceId, volumeId, scUrl);
    }

    if (connect_ebs_volume(scUrl, attachmentToken, nc->config_use_ws_sec, nc->config_sc_policy_file, nc->ip, nc->iqn, &remoteDevStr, &vol_data) != EUCA_OK) {
        LOGERROR("Error connecting ebs volume %s\n", attachmentToken);
        have_remote_device = 0;
        ret = EUCA_ERROR;
        goto release;
    }

    if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
        LOGERROR("[%s][%s] failed to connect to iscsi target\n", instanceId, volumeId);
        remoteDevReal[0] = '\0';
        have_remote_device = 0;
        ret = EUCA_ERROR;
        goto release;
    } else {
        LOGDEBUG("[%s][%s] attached iSCSI target of host device '%s'\n", instanceId, volumeId, remoteDevStr);
        snprintf(remoteDevReal, sizeof(remoteDevReal), "%s", remoteDevStr);
        have_remote_device = 1;
    }

    // Update volume with new connection info
    sem_p(inst_sem);
    {
        volume = save_volume(instance, volumeId, attachmentToken, vol_data->connect_string, localDevName, localDevReal, VOL_STATE_ATTACHING);
        save_instance_struct(instance);
        copy_instances();
    }
    sem_v(inst_sem);

    if (!volume) {
        LOGERROR("[%s][%s] failed to update the volume record, aborting volume attachment\n", instanceId, volumeId);
        ret = EUCA_ERROR;
        goto release;
    }
    // make sure there is a block device
    if (check_block(remoteDevReal)) {
        LOGERROR("[%s][%s] cannot verify that host device '%s' is available for hypervisor attach\n", instanceId, volumeId, remoteDevReal);
        ret = EUCA_ERROR;
        goto release;
    }
    // generate XML for libvirt attachment request
    if (gen_volume_xml(volumeId, instance, localDevReal, remoteDevReal) // creates vol-XXX.xml
        || gen_libvirt_volume_xml(volumeId, instance)) {    // creates vol-XXX-libvirt.xml via XSLT transform
        LOGERROR("[%s][%s] could not produce attach device xml\n", instanceId, volumeId);
        ret = EUCA_ERROR;
        goto release;
    }
    // invoke hooks
    char path[MAX_PATH];
    char lpath[MAX_PATH];
    snprintf(path, sizeof(path), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volumeId);  // vol-XXX.xml
    snprintf(lpath, sizeof(lpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volumeId);    // vol-XXX-libvirt.xml
    if (call_hooks(NC_EVENT_PRE_ATTACH, lpath)) {
        LOGERROR("[%s][%s] cancelled volume attachment via hooks\n", instance->instanceId, volumeId);
        ret = EUCA_ERROR;
        goto release;
    }
    // read in libvirt XML, which may have been modified by the hook above
    if ((xml = file2str(lpath)) == NULL) {
        LOGERROR("[%s][%s] failed to read volume XML from %s\n", instance->instanceId, volumeId, lpath);
        ret = EUCA_ERROR;
        goto release;
    }

    {                                  // connect to hypervisor, find the domain, attach the volume
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            return EUCA_HYPERVISOR_ERROR;
        }
        // find domain on hypervisor
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        if (dom == NULL) {
            if (instance->state != BOOTING && instance->state != STAGING) {
                LOGWARN("[%s][%s] domain not running on hypervisor, cannot attach device\n", instanceId, volumeId);
            }
            unlock_hypervisor_conn();
            goto release;
        }

        int err = 0;
        for (int i = 1; i <= VOL_RETRIES; i++) {
            err = virDomainAttachDevice(dom, xml);
            if (err) {
                LOGERROR("[%s][%s] failed to attach host device '%s' to guest device '%s' on attempt %d of 3\n", instanceId, volumeId, remoteDevReal, localDevReal, i);
                LOGDEBUG("[%s][%s] virDomainAttachDevice() failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
                sleep(3);              // sleep a bit and retry.
            } else {
                break;
            }
        }

        if (err) {
            LOGERROR("[%s][%s] failed to attach host device '%s' to guest device '%s' after %d tries\n", instanceId, volumeId, remoteDevReal, localDevReal, VOL_RETRIES);
            LOGDEBUG("[%s][%s] virDomainAttachDevice() failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
            ret = EUCA_ERROR;
        }

        virDomainFree(dom);            // release libvirt resource
        unlock_hypervisor_conn();
    }

release:

    {                                  // record volume state in memory and on disk
        char *next_vol_state;
        if (ret == EUCA_OK) {
            next_vol_state = VOL_STATE_ATTACHED;
        } else {
            next_vol_state = VOL_STATE_ATTACHING_FAILED;
        }

        sem_p(inst_sem);
        {
            volume = save_volume(instance, volumeId, NULL, NULL, NULL, NULL, next_vol_state);   // now we can record remoteDevReal
            save_instance_struct(instance);
            copy_instances();
            update_disk_aliases(instance);  // ask sensor subsystem to track the volume
        }
        sem_v(inst_sem);
    }

    if ((volume == NULL) && (xml != NULL)) {
        LOGERROR("[%s][%s] failed to save the volume record, aborting volume attachment (detaching)\n", instanceId, volumeId);

        // connect to hypervisor, find the domain, detach the volume
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            return EUCA_HYPERVISOR_ERROR;
        }
        // find domain on hypervisor
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        int err = 0;
        if (dom != NULL) {
            err = virDomainDetachDevice(dom, xml);
            virDomainFree(dom);        // release libvirt resource
        }
        if (err) {
            LOGERROR("[%s][%s] failed to detach as part of aborting\n", instanceId, volumeId);
            LOGDEBUG("[%s][%s] virDomainDetachDevice() failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
        }
        unlock_hypervisor_conn();
        ret = EUCA_ERROR;
    }
    // if iSCSI and there were problems, try to disconnect the target
    if (ret != EUCA_OK && have_remote_device) {
        LOGDEBUG("[%s][%s] attempting to disconnect iscsi target due to attachment failure\n", instanceId, volumeId);
        if (vol_data != NULL && vol_data->connect_string[0] != '\0') {
            if (disconnect_ebs_volume(scUrl, nc->config_use_ws_sec, nc->config_sc_policy_file, attachmentToken, vol_data->connect_string, nc->ip, nc->iqn) != EUCA_OK) {
                LOGERROR("[%s][%s] Error disconnecting ebs volume on error rollback.\n", instanceId, volumeId);
            }
        }
    }

    if (ret == EUCA_OK) {
        LOGINFO("[%s][%s] volume attached as host device '%s' to guest device '%s'\n", instanceId, volumeId, remoteDevReal, localDevReal);
    }
    // remoteDev can be a long string, so to keep log readable, we log it at TRACE level unless there was a problem
    int log_level_for_devstring = EUCA_LOG_TRACE;
    if (ret != EUCA_OK)
        log_level_for_devstring = EUCA_LOG_DEBUG;
    EUCALOG(log_level_for_devstring, "[%s][%s] remote device string: %s\n", instanceId, volumeId, remoteDevStr);

    if (vol_data)
        EUCA_FREE(vol_data);
    if (remoteDevStr)
        EUCA_FREE(remoteDevStr);

    EUCA_FREE(xml);

    return ret;
}
static int doAttachVolume(struct nc_state_t *nc, ncMetadata * pMeta, char *instanceId, char *volumeId, char *attachmentToken, char *localDev)
{
    int ret = EUCA_OK;
    int have_remote_device = 0;
    char *xml = NULL;
    char *tagBuf;
    char *localDevName;
    char localDevReal[32], localDevTag[256], remoteDevReal[132];
    char scUrl[512];                    
    ncVolume *volume;
    ebs_volume_data *vol_data = NULL;
    char *remoteDevStr = NULL;

    if (!strcmp(nc->H->name, "xen")) {
        tagBuf = NULL;
        localDevName = localDevReal;
    } else if (!strcmp(nc->H->name, "kvm")) {
        tagBuf = localDevTag;
        localDevName = localDevTag;
    } else {
        LOGERROR("[%s][%s] unknown hypervisor type '%s'\n", instanceId, volumeId, nc->H->name);
        return EUCA_ERROR;
    }

     
     
    ret = convert_dev_names(localDev, localDevReal, tagBuf);
    if (ret)
        return ret;

     
    sem_p(inst_sem);
    ncInstance *instance = find_instance(&global_instances, instanceId);
    sem_v(inst_sem);
    if (instance == NULL) {
        return EUCA_NOT_FOUND_ERROR;
    }

    {                                   
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            return EUCA_HYPERVISOR_ERROR;
        }
         
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        if (dom == NULL) {
            if (instance->state != BOOTING && instance->state != STAGING) {
                LOGWARN("[%s][%s] domain not running on hypervisor, cannot attach device\n", instanceId, volumeId);
            }
            unlock_hypervisor_conn();
            return EUCA_HYPERVISOR_ERROR;
        }
        virDomainFree(dom);             
        unlock_hypervisor_conn();
    }

     
    sem_p(inst_sem);
    {
        volume = save_volume(instance, volumeId, attachmentToken, NULL, localDevName, localDevReal, VOL_STATE_ATTACHING);
        save_instance_struct(instance);
        copy_instances();
    }
    sem_v(inst_sem);

    if (!volume) {
        LOGERROR("[%s][%s] failed to update the volume record, aborting volume attachment\n", instanceId, volumeId);
        return EUCA_ERROR;
    }
     
    LOGTRACE("[%s][%s] Connecting EBS volume to local host\n", instanceId, volumeId);
    if (get_service_url("storage", nc, scUrl) != EUCA_OK || strlen(scUrl) == 0) {
        LOGERROR("[%s][%s] Failed to lookup enabled Storage Controller. Cannot attach volume: %s\n", instanceId, volumeId, volumeId);
        have_remote_device = 0;
        ret = EUCA_ERROR;
        goto release;
    } else {
        LOGTRACE("[%s][%s] Using SC URL: %s\n", instanceId, volumeId, scUrl);
    }

    if (connect_ebs_volume(scUrl, attachmentToken, nc->config_use_ws_sec, nc->config_sc_policy_file, nc->ip, nc->iqn, &remoteDevStr, &vol_data) != EUCA_OK) {
        LOGERROR("Error connecting ebs volume %s\n", attachmentToken);
        have_remote_device = 0;
        ret = EUCA_ERROR;
        goto release;
    }

    if (!remoteDevStr || !strstr(remoteDevStr, "/dev")) {
        LOGERROR("[%s][%s] failed to connect to iscsi target\n", instanceId, volumeId);
        remoteDevReal[0] = '\0';
        have_remote_device = 0;
        ret = EUCA_ERROR;
        goto release;
    } else {
        LOGDEBUG("[%s][%s] attached iSCSI target of host device '%s'\n", instanceId, volumeId, remoteDevStr);
        snprintf(remoteDevReal, sizeof(remoteDevReal), "%s", remoteDevStr);
        have_remote_device = 1;
    }

     
    sem_p(inst_sem);
    {
        volume = save_volume(instance, volumeId, attachmentToken, vol_data->connect_string, localDevName, localDevReal, VOL_STATE_ATTACHING);
        save_instance_struct(instance);
        copy_instances();
    }
    sem_v(inst_sem);

    if (!volume) {
        LOGERROR("[%s][%s] failed to update the volume record, aborting volume attachment\n", instanceId, volumeId);
        ret = EUCA_ERROR;
        goto release;
    }
     
    if (check_block(remoteDevReal)) {
        LOGERROR("[%s][%s] cannot verify that host device '%s' is available for hypervisor attach\n", instanceId, volumeId, remoteDevReal);
        ret = EUCA_ERROR;
        goto release;
    }
     
    if (gen_volume_xml(volumeId, instance, localDevReal, remoteDevReal)  
        || gen_libvirt_volume_xml(volumeId, instance)) {     
        LOGERROR("[%s][%s] could not produce attach device xml\n", instanceId, volumeId);
        ret = EUCA_ERROR;
        goto release;
    }
     
    char path[MAX_PATH];
    char lpath[MAX_PATH];
    snprintf(path, sizeof(path), EUCALYPTUS_VOLUME_XML_PATH_FORMAT, instance->instancePath, volumeId);   
    snprintf(lpath, sizeof(lpath), EUCALYPTUS_VOLUME_LIBVIRT_XML_PATH_FORMAT, instance->instancePath, volumeId);     
    if (call_hooks(NC_EVENT_PRE_ATTACH, lpath)) {
        LOGERROR("[%s][%s] cancelled volume attachment via hooks\n", instance->instanceId, volumeId);
        ret = EUCA_ERROR;
        goto release;
    }
     
    if ((xml = file2str(lpath)) == NULL) {
        LOGERROR("[%s][%s] failed to read volume XML from %s\n", instance->instanceId, volumeId, lpath);
        ret = EUCA_ERROR;
        goto release;
    }

    {                                   
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            return EUCA_HYPERVISOR_ERROR;
        }
         
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        if (dom == NULL) {
            if (instance->state != BOOTING && instance->state != STAGING) {
                LOGWARN("[%s][%s] domain not running on hypervisor, cannot attach device\n", instanceId, volumeId);
            }
            unlock_hypervisor_conn();
            goto release;
        }

        int err = 0;
        for (int i = 1; i <= VOL_RETRIES; i++) {
            err = virDomainAttachDevice(dom, xml);
            if (err) {
                LOGERROR("[%s][%s] failed to attach host device '%s' to guest device '%s' on attempt %d of 3\n", instanceId, volumeId, remoteDevReal, localDevReal, i);
                LOGDEBUG("[%s][%s] virDomainAttachDevice() failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
                sleep(3);               
            } else {
                break;
            }
        }

        if (err) {
            LOGERROR("[%s][%s] failed to attach host device '%s' to guest device '%s' after %d tries\n", instanceId, volumeId, remoteDevReal, localDevReal, VOL_RETRIES);
            LOGDEBUG("[%s][%s] virDomainAttachDevice() failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
            ret = EUCA_ERROR;
        }

        virDomainFree(dom);             
        unlock_hypervisor_conn();
    }

release:

    {                                   
        char *next_vol_state;
        if (ret == EUCA_OK) {
            next_vol_state = VOL_STATE_ATTACHED;
        } else {
            next_vol_state = VOL_STATE_ATTACHING_FAILED;
        }

        sem_p(inst_sem);
        {
            volume = save_volume(instance, volumeId, NULL, NULL, NULL, NULL, next_vol_state);    
            save_instance_struct(instance);
            copy_instances();
            update_disk_aliases(instance);   
        }
        sem_v(inst_sem);
    }

    if ((volume == NULL) && (xml != NULL)) {
        LOGERROR("[%s][%s] failed to save the volume record, aborting volume attachment (detaching)\n", instanceId, volumeId);

         
        virConnectPtr conn = lock_hypervisor_conn();
        if (conn == NULL) {
            LOGERROR("[%s][%s] cannot get connection to hypervisor\n", instanceId, volumeId);
            return EUCA_HYPERVISOR_ERROR;
        }
         
        virDomainPtr dom = virDomainLookupByName(conn, instanceId);
        int err = 0;
        if (dom != NULL) {
            err = virDomainDetachDevice(dom, xml);
            virDomainFree(dom);         
        }
        if (err) {
            LOGERROR("[%s][%s] failed to detach as part of aborting\n", instanceId, volumeId);
            LOGDEBUG("[%s][%s] virDomainDetachDevice() failed (err=%d) XML='%s'\n", instanceId, volumeId, err, xml);
        }
        unlock_hypervisor_conn();
        ret = EUCA_ERROR;
    }
     
    if (ret != EUCA_OK && have_remote_device) {
        LOGDEBUG("[%s][%s] attempting to disconnect iscsi target due to attachment failure\n", instanceId, volumeId);
        if (vol_data != NULL && vol_data->connect_string[0] != '\0') {
            if (disconnect_ebs_volume(scUrl, nc->config_use_ws_sec, nc->config_sc_policy_file, attachmentToken, vol_data->connect_string, nc->ip, nc->iqn) != EUCA_OK) {
                LOGERROR("[%s][%s] Error disconnecting ebs volume on error rollback.\n", instanceId, volumeId);
            }
        }
    }

    if (ret == EUCA_OK) {
        LOGINFO("[%s][%s] volume attached as host device '%s' to guest device '%s'\n", instanceId, volumeId, remoteDevReal, localDevReal);
    }
     
    int log_level_for_devstring = EUCA_LOG_TRACE;
    if (ret != EUCA_OK)
        log_level_for_devstring = EUCA_LOG_DEBUG;
    EUCALOG(log_level_for_devstring, "[%s][%s] remote device string: %s\n", instanceId, volumeId, remoteDevStr);

    if (vol_data)
        EUCA_FREE(vol_data);
    if (remoteDevStr)
        EUCA_FREE(remoteDevStr);

    EUCA_FREE(xml);

    return ret;
}