Program: 24959
Project: 24959
Buggy source:
/** 
 * Runs the scorer over the specified documents.
 * @throws IOException
 */
public void run(final SystemOutputStore systemAnswerStore,final AnnotationStore goldAnswerStore,final Set<Symbol> documentsToScore,final ScoringConfiguration scoringConfiguration,final File baseOutputDir) throws IOException {
  final Map<KBPScoringObserver<TypeRoleFillerRealis>,File> scorerToOutputDir=makeScorerToOutputDir(baseOutputDir,scoringConfiguration.scoringObservers());
  for (  final KBPScoringObserver<TypeRoleFillerRealis> observer : scoringConfiguration.scoringObservers()) {
    observer.startCorpus();
  }
  for (  final Symbol docid : documentsToScore) {
    log.info("String_Node_Str",docid);
    AnswerKey key=goldAnswerStore.readOrEmpty(docid);
    for (    Function<AnswerKey,AnswerKey> answerKeyTransformation : scoringConfiguration.answerKeyTransformations()) {
      key=answerKeyTransformation.apply(key);
    }
    final OnlyMostSpecificTemporal temporalFilter=OnlyMostSpecificTemporal.forAnswerKey(key);
    final EntityNormalizer entityNormalizer=EntityNormalizer.fromAnnotation(key);
    final AnswerKeyAnswerSource<TypeRoleFillerRealis> answerKey=AnswerKeyAnswerSource.forAnswerable(key,TypeRoleFillerRealis.extractFromSystemResponse(entityNormalizer));
    SystemOutput rawSystemOutput=systemAnswerStore.readOrEmpty(docid);
    for (    final Function<SystemOutput,SystemOutput> systemOutputTransformation : scoringConfiguration.systemOutputTransformations()) {
      rawSystemOutput=systemOutputTransformation.apply(rawSystemOutput);
    }
    final SystemOutputAnswerSource<TypeRoleFillerRealis> systemOutput=SystemOutputAnswerSource.forAnswerable(temporalFilter.apply(rawSystemOutput),TypeRoleFillerRealis.extractFromSystemResponse(entityNormalizer));
    final ImmutableMap<KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver,KBPScoringObserver<TypeRoleFillerRealis>> docObserversToCorpusObservers=documentObserversForCorpusObservers(scoringConfiguration.scoringObservers(),answerKey,systemOutput);
    final Set<KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver> docObservers=docObserversToCorpusObservers.keySet();
    for (    final KBPScoringObserver<?>.KBPAnswerSourceObserver observer : docObservers) {
      observer.start();
    }
    final Set<TypeRoleFillerRealis> allAnswerables=ImmutableSet.copyOf(concat(systemOutput.answerables(),answerKey.answerables()));
    final Ordering<TypeRoleFillerRealis> order=ByJustificationLocation.create(answerKey,systemOutput);
    for (    final TypeRoleFillerRealis answerable : order.sortedCopy(allAnswerables)) {
      log.info("String_Node_Str",answerable);
      for (      final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
        observer.startAnswerable(answerable);
      }
      final Set<Response> responses=systemOutput.answers(answerable);
      final Set<AssessedResponse> annotatedResponses=answerKey.answers(answerable);
      for (      final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
        observer.observe(answerable,responses,annotatedResponses);
      }
      if (!responses.isEmpty()) {
        final Response selectedSystemResponse=systemOutput.systemOutput().selectFromMultipleSystemResponses(responses).get();
        final Optional<AssessedResponse> annotationForArgument=AssessedResponse.findAnnotationForArgument(selectedSystemResponse,annotatedResponses);
        if (annotationForArgument.isPresent()) {
          for (          final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
            observer.annotatedSelectedResponse(answerable,selectedSystemResponse,annotationForArgument.get(),annotatedResponses);
          }
        }
 else {
          for (          final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
            observer.unannotatedSelectedResponse(answerable,selectedSystemResponse);
          }
        }
        if (annotatedResponses.isEmpty()) {
          for (          final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
            observer.responsesOnlyNonEmpty(answerable,responses);
          }
        }
      }
 else       if (!annotatedResponses.isEmpty()) {
        for (        final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
          observer.annotationsOnlyNonEmpty(answerable,annotatedResponses);
        }
      }
 else {
        throw new RuntimeException("String_Node_Str");
      }
      for (      final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
        observer.endAnswerable(answerable);
      }
    }
    for (    final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
      observer.end();
    }
    for (    final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
      File outputDir=new File(scorerToOutputDir.get(docObserversToCorpusObservers.get(observer)),docid.toString());
      outputDir.mkdirs();
      observer.writeDocumentOutput(outputDir);
    }
  }
  log.info("String_Node_Str");
  for (  final KBPScoringObserver<?> observer : scoringConfiguration.scoringObservers) {
    observer.endCorpus();
  }
  for (  final Map.Entry<KBPScoringObserver<TypeRoleFillerRealis>,File> corpusOutput : scorerToOutputDir.entrySet()) {
    corpusOutput.getKey().writeCorpusOutput(corpusOutput.getValue());
  }
}

Fixed version (ground truth):
/** 
 * Runs the scorer over the specified documents.
 * @throws IOException
 */
public void run(final SystemOutputStore systemAnswerStore,final AnnotationStore goldAnswerStore,final Set<Symbol> documentsToScore,final ScoringConfiguration scoringConfiguration,final File baseOutputDir) throws IOException {
  final Map<KBPScoringObserver<TypeRoleFillerRealis>,File> scorerToOutputDir=makeScorerToOutputDir(baseOutputDir,scoringConfiguration.scoringObservers());
  for (  final KBPScoringObserver<TypeRoleFillerRealis> observer : scoringConfiguration.scoringObservers()) {
    observer.startCorpus();
  }
  for (  final Symbol docid : documentsToScore) {
    log.info("String_Node_Str",docid);
    AnswerKey key=goldAnswerStore.readOrEmpty(docid);
    for (    Function<AnswerKey,AnswerKey> answerKeyTransformation : scoringConfiguration.answerKeyTransformations()) {
      key=answerKeyTransformation.apply(key);
    }
    final OnlyMostSpecificTemporal temporalFilter=OnlyMostSpecificTemporal.forAnswerKey(key);
    final EntityNormalizer entityNormalizer=EntityNormalizer.fromAnnotation(key);
    final AnswerKeyAnswerSource<TypeRoleFillerRealis> answerKey=AnswerKeyAnswerSource.forAnswerable(key,TypeRoleFillerRealis.extractFromSystemResponse(entityNormalizer));
    SystemOutput rawSystemOutput=systemAnswerStore.readOrEmpty(docid);
    for (    final Function<SystemOutput,SystemOutput> systemOutputTransformation : scoringConfiguration.systemOutputTransformations()) {
      rawSystemOutput=systemOutputTransformation.apply(rawSystemOutput);
    }
    final SystemOutputAnswerSource<TypeRoleFillerRealis> systemOutput=SystemOutputAnswerSource.forAnswerable(temporalFilter.apply(rawSystemOutput),TypeRoleFillerRealis.extractFromSystemResponse(entityNormalizer));
    final ImmutableMap<KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver,KBPScoringObserver<TypeRoleFillerRealis>> docObserversToCorpusObservers=documentObserversForCorpusObservers(scoringConfiguration.scoringObservers(),answerKey,systemOutput);
    final Set<KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver> docObservers=docObserversToCorpusObservers.keySet();
    for (    final KBPScoringObserver<?>.KBPAnswerSourceObserver observer : docObservers) {
      observer.start();
    }
    final Set<TypeRoleFillerRealis> allAnswerables=ImmutableSet.copyOf(concat(systemOutput.answerables(),answerKey.answerables()));
    final Ordering<TypeRoleFillerRealis> order=ByJustificationLocation.create(answerKey,systemOutput);
    for (    final TypeRoleFillerRealis answerable : order.sortedCopy(allAnswerables)) {
      log.info("String_Node_Str",answerable);
      for (      final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
        observer.startAnswerable(answerable);
      }
      final Set<Response> responses=systemOutput.answers(answerable);
      final Set<AssessedResponse> annotatedResponses=answerKey.answers(answerable);
      for (      final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
        observer.observe(answerable,responses,annotatedResponses);
      }
      if (!responses.isEmpty()) {
        final Response selectedSystemResponse=systemOutput.systemOutput().selectFromMultipleSystemResponses(responses).get();
        final Optional<AssessedResponse> annotationForArgument=AssessedResponse.findAnnotationForArgument(selectedSystemResponse,annotatedResponses);
        if (annotationForArgument.isPresent()) {
          for (          final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
            observer.annotatedSelectedResponse(answerable,selectedSystemResponse,annotationForArgument.get(),annotatedResponses);
          }
        }
 else {
          for (          final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
            observer.unannotatedSelectedResponse(answerable,selectedSystemResponse,annotatedResponses);
          }
        }
        if (annotatedResponses.isEmpty()) {
          for (          final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
            observer.responsesOnlyNonEmpty(answerable,responses);
          }
        }
      }
 else       if (!annotatedResponses.isEmpty()) {
        for (        final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
          observer.annotationsOnlyNonEmpty(answerable,annotatedResponses);
        }
      }
 else {
        throw new RuntimeException("String_Node_Str");
      }
      for (      final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
        observer.endAnswerable(answerable);
      }
    }
    for (    final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
      observer.end();
    }
    for (    final KBPScoringObserver<TypeRoleFillerRealis>.KBPAnswerSourceObserver observer : docObservers) {
      File outputDir=new File(scorerToOutputDir.get(docObserversToCorpusObservers.get(observer)),docid.toString());
      outputDir.mkdirs();
      observer.writeDocumentOutput(outputDir);
    }
  }
  log.info("String_Node_Str");
  for (  final KBPScoringObserver<?> observer : scoringConfiguration.scoringObservers) {
    observer.endCorpus();
  }
  for (  final Map.Entry<KBPScoringObserver<TypeRoleFillerRealis>,File> corpusOutput : scorerToOutputDir.entrySet()) {
    corpusOutput.getKey().writeCorpusOutput(corpusOutput.getValue());
  }
}