Source:
public static void purgeExpiredJobs(String workflow, WorkflowConfig workflowConfig, WorkflowContext workflowContext, HelixManager manager, RebalanceScheduler rebalanceScheduler) { if (workflowContext == null) { LOG.warn(String.format("Workflow %s context does not exist!", workflow)); return; } long purgeInterval = workflowConfig.getJobPurgeInterval(); long currentTime = System.currentTimeMillis(); final Set<String> expiredJobs = Sets.newHashSet(); if (purgeInterval > 0 && workflowContext.getLastJobPurgeTime() + purgeInterval <= currentTime) { expiredJobs.addAll(TaskUtil.getExpiredJobs(manager.getHelixDataAccessor(), manager.getHelixPropertyStore(), workflowConfig, workflowContext)); if (expiredJobs.isEmpty()) { LOG.info("No job to purge for the queue " + workflow); } else { LOG.info("Purge jobs " + expiredJobs + " from queue " + workflow); Set<String> failedJobRemovals = new HashSet<>(); for (String job : expiredJobs) { if (!TaskUtil.removeJob(manager.getHelixDataAccessor(), manager.getHelixPropertyStore(), job)) { failedJobRemovals.add(job); LOG.warn("Failed to clean up expired and completed jobs from workflow " + workflow); } rebalanceScheduler.removeScheduledRebalance(job); } expiredJobs.removeAll(failedJobRemovals); if (!TaskUtil.removeJobsFromDag(manager.getHelixDataAccessor(), workflow, expiredJobs, true)) { LOG.warn("Error occurred while trying to remove jobs + " + expiredJobs + " from the workflow " + workflow); } if (expiredJobs.size() > 0) { HelixDataAccessor accessor = manager.getHelixDataAccessor(); List<String> resourceConfigs = accessor.getChildNames(accessor.keyBuilder().resourceConfigs()); if (resourceConfigs.size() > 0) { RebalanceUtil.scheduleOnDemandPipeline(manager.getClusterName(), 0); } else { LOG.warn( "No resource config to trigger rebalance for clean up contexts for" + expiredJobs); } } } } setNextJobPurgeTime(workflow, currentTime, purgeInterval, rebalanceScheduler, manager); }

Target:
public static void purgeExpiredJobs(String workflow, WorkflowConfig workflowConfig, WorkflowContext workflowContext, HelixManager manager, RebalanceScheduler rebalanceScheduler) { if (workflowContext == null) { LOG.warn(String.format("Workflow %s context does not exist!", workflow)); return; } long purgeInterval = workflowConfig.getJobPurgeInterval(); long currentTime = System.currentTimeMillis(); final Set<String> expiredJobs = Sets.newHashSet(); if (purgeInterval > 0 && workflowContext.getLastJobPurgeTime() + purgeInterval <= currentTime) { expiredJobs.addAll(TaskUtil.getExpiredJobs(manager.getHelixDataAccessor(), manager.getHelixPropertyStore(), workflowConfig, workflowContext)); if (expiredJobs.isEmpty()) { LOG.info("No job to purge for the queue " + workflow); } else { LOG.info("Purge jobs " + expiredJobs + " from queue " + workflow); Set<String> failedJobRemovals = new HashSet<>(); for (String job : expiredJobs) { if (!TaskUtil.removeJob(manager.getHelixDataAccessor(), manager.getHelixPropertyStore(), job)) { failedJobRemovals.add(job); LOG.warn("Failed to clean up expired and completed jobs from workflow " + workflow); } rebalanceScheduler.removeScheduledRebalance(job); } expiredJobs.removeAll(failedJobRemovals); if (!TaskUtil.removeJobsFromDag(manager.getHelixDataAccessor(), workflow, expiredJobs, true)) { LOG.warn("Error occurred while trying to remove jobs + " + expiredJobs + " from the workflow " + workflow); } if (expiredJobs.size() > 0) { HelixDataAccessor accessor = manager.getHelixDataAccessor(); List<String> resourceConfigs = accessor.getChildNames(accessor.keyBuilder().resourceConfigs()); if (resourceConfigs.size() > 0) { RebalanceUtil.scheduleOnDemandPipeline(manager.getClusterName(), 0L); } else { LOG.warn( "No resource config to trigger rebalance for clean up contexts for" + expiredJobs); } } } } setNextJobPurgeTime(workflow, currentTime, purgeInterval, rebalanceScheduler, manager); }