Program: 36069
Project: 36069
Buggy source:
/** 
 * Full evaluation of guesses to gold links. This penalizes guesses for not labeling everything. The goldDocs and guessedDocs should cover the same docs.
 * @param goldDocs Gold tlinks in every document.
 * @param guessedDocs The guessed tlinks in every document.
 */
public static void evaluate(SieveDocuments goldDocs,SieveDocuments guessedDocs){
  Counter<String> guessCounts=new ClassicCounter<String>();
  Counter<TLink.Type> goldLabelCounts=new ClassicCounter<TLink.Type>();
  int numCorrect=0;
  int numIncorrect=0;
  int numIncorrectNonVague=0;
  int numMissed=0;
  if (goldDocs == null)   return;
  normalizeAllTlinksByTextOrder(guessedDocs);
  for (  SieveDocument guessedDoc : guessedDocs.getDocuments()) {
    SieveDocument goldDoc=goldDocs.getDocument(guessedDoc.getDocname());
    Set<String> seenGoldLinks=new HashSet<String>();
    List<TLink> goldLinks=goldDoc.getTlinks(true);
    Map<String,TLink> goldPairLookup=new HashMap<String,TLink>();
    for (    TLink tlink : goldLinks)     goldPairLookup.put(tlink.getId1() + "String_Node_Str" + tlink.getId2(),tlink);
    List<TLink> proposed=guessedDoc.getTlinks();
    for (    TLink pp : proposed) {
      TLink goldLink=goldPairLookup.get(pp.getId1() + "String_Node_Str" + pp.getId2());
      if (goldLink != null) {
        guessCounts.incrementCount(goldLink.getRelation() + "String_Node_Str" + pp.getRelation());
        goldLabelCounts.incrementCount(goldLink.getRelation());
        seenGoldLinks.add(pp.getId1() + "String_Node_Str" + pp.getId2());
      }
      if (Evaluate.isLinkCorrect(pp,goldLinks)) {
        numCorrect++;
      }
 else       if (goldLink != null) {
        if (!goldLink.getRelation().equals(TLink.Type.VAGUE)) {
          numIncorrectNonVague++;
        }
        numIncorrect++;
      }
 else {
      }
    }
    for (    TLink gold : goldLinks) {
      if (!seenGoldLinks.contains(gold.getId1() + "String_Node_Str" + gold.getId2())) {
        numMissed++;
      }
    }
  }
  int totalGuessed=numCorrect + numIncorrect;
  int totalGold=numCorrect + numIncorrect + numMissed;
  double precision=(totalGuessed > 0 ? (double)numCorrect / totalGuessed : 0.0);
  double recall=(totalGold > 0 ? (double)numCorrect / totalGold : 0.0);
  double f1=(precision + recall > 0 ? 2.0 * precision * recall / (precision + recall) : 0.0);
  int totalGuessedNonVague=numCorrect + numIncorrectNonVague;
  double precisionNonVague=(totalGuessedNonVague > 0 ? numCorrect / totalGuessedNonVague : 0.0);
  System.out.printf("String_Node_Str",precision,numCorrect,totalGuessed,recall,numCorrect,totalGold,f1);
  System.out.printf("String_Node_Str",precisionNonVague,numCorrect,totalGuessedNonVague);
  printBaseline(goldLabelCounts);
  confusionMatrix(guessCounts);
}

Fixed version (ground truth):
/** 
 * Full evaluation of guesses to gold links. This penalizes guesses for not labeling everything. The goldDocs and guessedDocs should cover the same docs.
 * @param goldDocs Gold tlinks in every document.
 * @param guessedDocs The guessed tlinks in every document.
 */
public static void evaluate(SieveDocuments goldDocs,SieveDocuments guessedDocs,Map<String,SieveStats> sieveStats){
  Counter<String> guessCounts=new ClassicCounter<String>();
  Counter<TLink.Type> goldLabelCounts=new ClassicCounter<TLink.Type>();
  int numCorrect=0;
  int numIncorrect=0;
  int numIncorrectNonVague=0;
  int numMissed=0;
  if (goldDocs == null)   return;
  normalizeAllTlinksByTextOrder(guessedDocs);
  for (  SieveDocument guessedDoc : guessedDocs.getDocuments()) {
    SieveDocument goldDoc=goldDocs.getDocument(guessedDoc.getDocname());
    Set<String> seenGoldLinks=new HashSet<String>();
    List<TLink> goldLinks=goldDoc.getTlinksNoClosures();
    Map<String,TLink> goldPairLookup=new HashMap<String,TLink>();
    for (    TLink tlink : goldLinks)     goldPairLookup.put(tlink.getId1() + "String_Node_Str" + tlink.getId2(),tlink);
    List<TLink> proposed=guessedDoc.getTlinks();
    for (    TLink pp : proposed) {
      TLink goldLink=goldPairLookup.get(pp.getId1() + "String_Node_Str" + pp.getId2());
      if (goldLink != null) {
        guessCounts.incrementCount(goldLink.getRelation() + "String_Node_Str" + pp.getRelation());
        goldLabelCounts.incrementCount(goldLink.getRelation());
        seenGoldLinks.add(pp.getId1() + "String_Node_Str" + pp.getId2());
      }
      if (Evaluate.isLinkCorrect(pp,goldLinks)) {
        numCorrect++;
        if (pp.getOrigin() != null) {
          sieveStats.get(pp.getOrigin()).addCorrect(pp);
        }
 else         System.out.println("String_Node_Str" + pp);
      }
 else       if (goldLink != null) {
        if (!goldLink.getRelation().equals(TLink.Type.VAGUE)) {
          numIncorrectNonVague++;
        }
        numIncorrect++;
        if (pp.getOrigin() != null)         sieveStats.get(pp.getOrigin()).addIncorrect(pp,goldLink);
 else         System.out.println("String_Node_Str" + pp);
      }
 else {
        sieveStats.get(pp.getOrigin()).addNoGold(pp);
      }
    }
    for (    TLink gold : goldLinks) {
      if (!seenGoldLinks.contains(gold.getId1() + "String_Node_Str" + gold.getId2())) {
        numMissed++;
      }
    }
  }
  int totalGuessed=numCorrect + numIncorrect;
  int totalGold=numCorrect + numIncorrect + numMissed;
  double precision=(totalGuessed > 0 ? (double)numCorrect / totalGuessed : 0.0);
  double recall=(totalGold > 0 ? (double)numCorrect / totalGold : 0.0);
  double f1=(precision + recall > 0 ? 2.0 * precision * recall / (precision + recall) : 0.0);
  int totalGuessedNonVague=numCorrect + numIncorrectNonVague;
  double precisionNonVague=(totalGuessedNonVague > 0 ? (double)numCorrect / (double)totalGuessedNonVague : 0.0);
  for (  SieveStats ss : sieveStats.values())   ss.printStats();
  System.out.println("String_Node_Str");
  System.out.println("String_Node_Str");
  System.out.println("String_Node_Str");
  System.out.printf("String_Node_Str",precision,numCorrect,totalGuessed,recall,numCorrect,totalGold,f1);
  System.out.printf("String_Node_Str",precisionNonVague,numCorrect,totalGuessedNonVague);
  System.out.println();
  printBaseline(goldLabelCounts);
  confusionMatrix(guessCounts);
  System.out.println("String_Node_Str");
}