Program: 49287
Project: 49287
Buggy source:
/** 
 * Some History, hopefully irrelavant henceforth: This transaction used to update consumer's status hash and got dead locked because: T1 and T2 are entitlement jobs 1. T1 grabs a shared lock on cp_consumer.id due to the FK in cp_entitlement when inserting into cp_entitlement 2. T2 grabs a shared lock on cp_consumer.id due to the FK in cp_entitlement when inserting into cp_entitlement 3. T1 attempts to grab an exclusive lock on cp_consumer.id for an update to cp_consumer's compliance hash.  T1 blocks waiting for the T2's shared lock to be released. 4. T2 attempts to grab an exclusive lock on cp_consumer.id for an update to cp_consumer's compliance hash. 5. Deadlock.  T2 is waiting for T1's shared lock to be released but T1 is waiting for T2's shared lock to be released. The solution was to create a longer transaction and grab an exclusive lock on the cp_consumer row (using a select for update) at the start of the transaction. The other thread will then wait for the exclusive lock to be released instead of deadlocking. Another effort on the solution removed compliance status evaluation from this thread and created a separate asynchronous job to accomplish that, thus removing the need to hold that exclusive lock on cp_consumer See BZ #1274074 and git history for details
 */
@Transactional protected List<Entitlement> addOrUpdateEntitlements(Consumer consumer,Map<String,Integer> poolQuantityMap,Map<String,Entitlement> entitlements,boolean generateUeberCert,CallerType caller) throws EntitlementRefusedException {
  log.debug("String_Node_Str",poolQuantityMap.keySet());
  List<Pool> pools=poolCurator.lockAndLoad(poolQuantityMap.keySet());
  for (  Pool pool : pools) {
    log.debug("String_Node_Str",pool,pool.getConsumed());
  }
  Map<String,PoolQuantity> poolQuantities=new HashMap<String,PoolQuantity>();
  boolean quantityFound=false;
  for (  Pool pool : pools) {
    Integer quantity=poolQuantityMap.get(pool.getId());
    if (quantity > 0) {
      quantityFound=true;
    }
    poolQuantities.put(pool.getId(),new PoolQuantity(pool,quantity));
    poolQuantityMap.remove(pool.getId());
  }
  if (!poolQuantityMap.isEmpty()) {
    throw new IllegalArgumentException(i18n.tr("String_Node_Str",poolQuantityMap.keySet()));
  }
  if (quantityFound) {
    log.info("String_Node_Str");
    Map<String,ValidationResult> results=enforcer.preEntitlement(consumer,poolQuantities.values(),caller);
    for (    Entry<String,ValidationResult> entry : results.entrySet()) {
      ValidationResult result=entry.getValue();
      if (!result.isSuccessful()) {
        log.warn("String_Node_Str",result.getErrors().toString(),entry.getKey());
        throw new EntitlementRefusedException(results);
      }
    }
  }
  EntitlementHandler handler=null;
  if (entitlements == null) {
    handler=new NewHandler();
  }
 else   if (entitlements.keySet().size() != poolQuantities.keySet().size() || !entitlements.keySet().containsAll(poolQuantities.keySet()) || !poolQuantities.keySet().containsAll(entitlements.keySet())) {
    throw new IllegalArgumentException(i18n.tr("String_Node_Str" + "String_Node_Str",entitlements.keySet().size(),poolQuantityMap.keySet().size()));
  }
 else {
    handler=new UpdateHandler();
  }
  log.info("String_Node_Str");
  entitlements=handler.handleEntitlement(consumer,poolQuantities,entitlements);
  log.info("String_Node_Str");
  handler.handleEntitlementPersist(entitlements);
  for (  Entry<String,PoolQuantity> entry : poolQuantities.entrySet()) {
    Pool pool=entry.getValue().getPool();
    Integer quantity=entry.getValue().getQuantity();
    pool.setConsumed(pool.getConsumed() + quantity);
    if (consumer.getType().isManifest()) {
      pool.setExported(pool.getExported() + quantity);
    }
  }
  handler.handlePostEntitlement(this,consumer,entitlements);
  handler.handleSelfCertificates(consumer,poolQuantities,entitlements,generateUeberCert);
  for (  Entitlement regenEnt : entitlementCurator.listModifying(entitlements.values())) {
    this.regenerateCertificatesOf(regenEnt,generateUeberCert,true);
  }
  handler.handleBonusPools(poolQuantities,entitlements);
  JobDetail detail=ConsumerComplianceJob.scheduleStatusCheck(consumer,null,false,false);
  log.info("String_Node_Str",detail.getKey(),consumer.getUuid());
  return new ArrayList<Entitlement>(entitlements.values());
}

Fixed version (ground truth):
/** 
 * Some History, hopefully irrelavant henceforth: This transaction used to update consumer's status hash and got dead locked because: T1 and T2 are entitlement jobs 1. T1 grabs a shared lock on cp_consumer.id due to the FK in cp_entitlement when inserting into cp_entitlement 2. T2 grabs a shared lock on cp_consumer.id due to the FK in cp_entitlement when inserting into cp_entitlement 3. T1 attempts to grab an exclusive lock on cp_consumer.id for an update to cp_consumer's compliance hash.  T1 blocks waiting for the T2's shared lock to be released. 4. T2 attempts to grab an exclusive lock on cp_consumer.id for an update to cp_consumer's compliance hash. 5. Deadlock.  T2 is waiting for T1's shared lock to be released but T1 is waiting for T2's shared lock to be released. The solution was to create a longer transaction and grab an exclusive lock on the cp_consumer row (using a select for update) at the start of the transaction. The other thread will then wait for the exclusive lock to be released instead of deadlocking. Another effort on the solution removed compliance status evaluation from this thread and created a separate asynchronous job to accomplish that, thus removing the need to hold that exclusive lock on cp_consumer See BZ #1274074 and git history for details
 */
@Transactional protected List<Entitlement> addOrUpdateEntitlements(Consumer consumer,Map<String,Integer> poolQuantityMap,Map<String,Entitlement> entitlements,boolean generateUeberCert,CallerType caller) throws EntitlementRefusedException {
  log.debug("String_Node_Str",poolQuantityMap.keySet());
  List<Pool> pools=poolCurator.lockAndLoad(poolQuantityMap.keySet());
  for (  Pool pool : pools) {
    log.debug("String_Node_Str",pool,pool.getConsumed());
  }
  Map<String,PoolQuantity> poolQuantities=new HashMap<String,PoolQuantity>();
  boolean quantityFound=false;
  for (  Pool pool : pools) {
    Integer quantity=poolQuantityMap.get(pool.getId());
    if (quantity > 0) {
      quantityFound=true;
    }
    poolQuantities.put(pool.getId(),new PoolQuantity(pool,quantity));
    poolQuantityMap.remove(pool.getId());
  }
  if (!poolQuantityMap.isEmpty()) {
    throw new IllegalArgumentException(i18n.tr("String_Node_Str",poolQuantityMap.keySet()));
  }
  if (quantityFound) {
    log.info("String_Node_Str");
    Map<String,ValidationResult> results=enforcer.preEntitlement(consumer,poolQuantities.values(),caller);
    for (    Entry<String,ValidationResult> entry : results.entrySet()) {
      ValidationResult result=entry.getValue();
      if (!result.isSuccessful()) {
        log.warn("String_Node_Str",result.getErrors().toString(),entry.getKey());
        throw new EntitlementRefusedException(results);
      }
    }
  }
  EntitlementHandler handler=null;
  if (entitlements == null) {
    handler=new NewHandler();
  }
 else   if (entitlements.keySet().size() != poolQuantities.keySet().size() || !entitlements.keySet().containsAll(poolQuantities.keySet()) || !poolQuantities.keySet().containsAll(entitlements.keySet())) {
    throw new IllegalArgumentException(i18n.tr("String_Node_Str" + "String_Node_Str",entitlements.keySet().size(),poolQuantityMap.keySet().size()));
  }
 else {
    handler=new UpdateHandler();
  }
  log.info("String_Node_Str");
  entitlements=handler.handleEntitlement(consumer,poolQuantities,entitlements);
  log.info("String_Node_Str");
  handler.handleEntitlementPersist(entitlements);
  for (  Entry<String,PoolQuantity> entry : poolQuantities.entrySet()) {
    Pool pool=entry.getValue().getPool();
    Integer quantity=entry.getValue().getQuantity();
    pool.setConsumed(pool.getConsumed() + quantity);
    if (consumer.getType().isManifest()) {
      pool.setExported(pool.getExported() + quantity);
    }
  }
  handler.handlePostEntitlement(this,consumer,entitlements);
  handler.handleSelfCertificates(consumer,poolQuantities,entitlements,generateUeberCert);
  for (  Entitlement regenEnt : entitlementCurator.listModifying(entitlements.values())) {
    this.regenerateCertificatesOf(regenEnt,generateUeberCert,true);
  }
  handler.handleBonusPools(poolQuantities,entitlements);
  JobDetail detail=ConsumerComplianceJob.scheduleStatusCheck(consumer,null,false,false);
  detail.getJobDataMap().put(PinsetterJobListener.PRINCIPAL_KEY,new SystemPrincipal());
  log.info("String_Node_Str",detail.getKey(),consumer.getUuid());
  try {
    pinsetterKernel.scheduleSingleJob(detail);
  }
 catch (  PinsetterException e) {
    log.error("String_Node_Str",e.getMessage());
  }
  poolCurator.flush();
  return new ArrayList<Entitlement>(entitlements.values());
}