Program: 106574
Project: 106574
Buggy source:
@Override @DB public void cleanupSecondaryStorage(boolean recurring){
  try {
    List<HostVO> secondaryStorageHosts=_hostDao.listSecondaryStorageHosts();
    for (    HostVO secondaryStorageHost : secondaryStorageHosts) {
      try {
        long hostId=secondaryStorageHost.getId();
        List<VMTemplateHostVO> destroyedTemplateHostVOs=_vmTemplateHostDao.listDestroyed(hostId);
        s_logger.debug("String_Node_Str" + destroyedTemplateHostVOs.size() + "String_Node_Str"+ secondaryStorageHost.getName());
        for (        VMTemplateHostVO destroyedTemplateHostVO : destroyedTemplateHostVOs) {
          if (!_tmpltMgr.templateIsDeleteable(destroyedTemplateHostVO)) {
            if (s_logger.isDebugEnabled()) {
              s_logger.debug("String_Node_Str" + destroyedTemplateHostVO);
            }
            continue;
          }
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("String_Node_Str" + destroyedTemplateHostVO);
          }
          String installPath=destroyedTemplateHostVO.getInstallPath();
          if (installPath != null) {
            Answer answer=_agentMgr.sendToSecStorage(secondaryStorageHost,new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(),destroyedTemplateHostVO.getInstallPath()));
            if (answer == null || !answer.getResult()) {
              s_logger.debug("String_Node_Str" + destroyedTemplateHostVO + "String_Node_Str"+ ((answer == null) ? "String_Node_Str" : answer.getDetails()));
            }
 else {
              _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId());
              s_logger.debug("String_Node_Str" + destroyedTemplateHostVO.getInstallPath());
            }
          }
 else {
            _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId());
          }
        }
      }
 catch (      Exception e) {
        s_logger.warn("String_Node_Str" + secondaryStorageHost,e);
      }
    }
    for (    HostVO secondaryStorageHost : secondaryStorageHosts) {
      try {
        long hostId=secondaryStorageHost.getId();
        List<Long> vIDs=findAllVolumeIdInSnapshotTable(hostId);
        if (vIDs == null) {
          continue;
        }
        for (        Long volumeId : vIDs) {
          boolean lock=false;
          try {
            VolumeVO volume=_volsDao.findByIdIncludingRemoved(volumeId);
            if (volume.getRemoved() == null) {
              volume=_volsDao.acquireInLockTable(volumeId,10);
              if (volume == null) {
                continue;
              }
              lock=true;
            }
            List<String> snapshots=findAllSnapshotForVolume(volumeId);
            if (snapshots == null) {
              continue;
            }
            CleanupSnapshotBackupCommand cmd=new CleanupSnapshotBackupCommand(secondaryStorageHost.getStorageUrl(),secondaryStorageHost.getDataCenterId(),volume.getAccountId(),volumeId,snapshots);
            Answer answer=_agentMgr.sendToSecStorage(secondaryStorageHost,cmd);
            if ((answer == null) || !answer.getResult()) {
              String details="String_Node_Str" + volumeId + "String_Node_Str"+ (answer == null ? "String_Node_Str" : answer.getDetails());
              s_logger.warn(details);
            }
          }
 catch (          Exception e1) {
            s_logger.warn("String_Node_Str" + secondaryStorageHost,e1);
          }
 finally {
            if (lock) {
              _volsDao.releaseFromLockTable(volumeId);
            }
          }
        }
      }
 catch (      Exception e2) {
        s_logger.warn("String_Node_Str" + secondaryStorageHost,e2);
      }
    }
  }
 catch (  Exception e3) {
    s_logger.warn("String_Node_Str",e3);
  }
}

Fixed version (ground truth):
@Override @DB public void cleanupSecondaryStorage(boolean recurring){
  try {
    List<HostVO> secondaryStorageHosts=_ssvmMgr.listSecondaryStorageHostsInAllZones();
    for (    HostVO secondaryStorageHost : secondaryStorageHosts) {
      try {
        long hostId=secondaryStorageHost.getId();
        List<VMTemplateHostVO> destroyedTemplateHostVOs=_vmTemplateHostDao.listDestroyed(hostId);
        s_logger.debug("String_Node_Str" + destroyedTemplateHostVOs.size() + "String_Node_Str"+ secondaryStorageHost.getName());
        for (        VMTemplateHostVO destroyedTemplateHostVO : destroyedTemplateHostVOs) {
          if (!_tmpltMgr.templateIsDeleteable(destroyedTemplateHostVO)) {
            if (s_logger.isDebugEnabled()) {
              s_logger.debug("String_Node_Str" + destroyedTemplateHostVO);
            }
            continue;
          }
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("String_Node_Str" + destroyedTemplateHostVO);
          }
          String installPath=destroyedTemplateHostVO.getInstallPath();
          if (installPath != null) {
            Answer answer=_agentMgr.sendToSecStorage(secondaryStorageHost,new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(),destroyedTemplateHostVO.getInstallPath()));
            if (answer == null || !answer.getResult()) {
              s_logger.debug("String_Node_Str" + destroyedTemplateHostVO + "String_Node_Str"+ ((answer == null) ? "String_Node_Str" : answer.getDetails()));
            }
 else {
              _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId());
              s_logger.debug("String_Node_Str" + destroyedTemplateHostVO.getInstallPath());
            }
          }
 else {
            _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId());
          }
        }
      }
 catch (      Exception e) {
        s_logger.warn("String_Node_Str" + secondaryStorageHost,e);
      }
    }
    for (    HostVO secondaryStorageHost : secondaryStorageHosts) {
      try {
        long hostId=secondaryStorageHost.getId();
        List<Long> vIDs=findAllVolumeIdInSnapshotTable(hostId);
        if (vIDs == null) {
          continue;
        }
        for (        Long volumeId : vIDs) {
          boolean lock=false;
          try {
            VolumeVO volume=_volsDao.findByIdIncludingRemoved(volumeId);
            if (volume.getRemoved() == null) {
              volume=_volsDao.acquireInLockTable(volumeId,10);
              if (volume == null) {
                continue;
              }
              lock=true;
            }
            List<String> snapshots=findAllSnapshotForVolume(volumeId);
            if (snapshots == null) {
              continue;
            }
            CleanupSnapshotBackupCommand cmd=new CleanupSnapshotBackupCommand(secondaryStorageHost.getStorageUrl(),secondaryStorageHost.getDataCenterId(),volume.getAccountId(),volumeId,snapshots);
            Answer answer=_agentMgr.sendToSecStorage(secondaryStorageHost,cmd);
            if ((answer == null) || !answer.getResult()) {
              String details="String_Node_Str" + volumeId + "String_Node_Str"+ (answer == null ? "String_Node_Str" : answer.getDetails());
              s_logger.warn(details);
            }
          }
 catch (          Exception e1) {
            s_logger.warn("String_Node_Str" + secondaryStorageHost,e1);
          }
 finally {
            if (lock) {
              _volsDao.releaseFromLockTable(volumeId);
            }
          }
        }
      }
 catch (      Exception e2) {
        s_logger.warn("String_Node_Str" + secondaryStorageHost,e2);
      }
    }
  }
 catch (  Exception e3) {
    s_logger.warn("String_Node_Str",e3);
  }
}