Program: 12289
Project: 12289
Buggy source:
/** 
 * reinforcement_learning.cpp:1193:rl_perform_update
 * @param i
 * @param g
 */
public void rl_perform_update(double op_value,IdentifierImpl goal){
  ReinforcementLearningInfo data=goal.rl_info;
  boolean using_gaps=(rl_get_parameter(RL_PARAM_TEMPORAL_EXTENSION,RL_RETURN_LONG) == RL_TE_ON);
  double alpha=rl_get_parameter(RL_PARAM_LEARNING_RATE);
  double lambda=rl_get_parameter(RL_PARAM_ET_DECAY_RATE);
  double gamma=rl_get_parameter(RL_PARAM_DISCOUNT_RATE);
  double tolerance=rl_get_parameter(RL_PARAM_ET_TOLERANCE);
  double update=data.reward;
  if (using_gaps)   update*=Math.pow(gamma,(double)data.reward_age);
  update+=(Math.pow(gamma,(double)data.step) * op_value);
  update-=data.previous_q;
  rl_set_stat(RL_STAT_UPDATE_ERROR,(double)(-update));
  if (lambda == 0) {
    if (!data.eligibility_traces.isEmpty())     data.eligibility_traces.clear();
  }
 else {
    Iterator<Map.Entry<Production,Double>> it=data.eligibility_traces.entrySet().iterator();
    while (it.hasNext()) {
      Map.Entry<Production,Double> e=it.next();
      e.setValue(e.getValue() * lambda);
      e.setValue(e.getValue() * Math.pow(gamma,data.step));
      if (e.getValue() < tolerance) {
        it.remove();
      }
    }
  }
  if (data.num_prev_op_rl_rules != 0) {
    double trace_increment=(1.0 / data.num_prev_op_rl_rules);
    for (    Production c : data.prev_op_rl_rules) {
      if (c != null) {
        Double iter=data.eligibility_traces.get(c);
        if (iter != null) {
          iter+=trace_increment;
          data.eligibility_traces.put(c,iter);
        }
 else {
          data.eligibility_traces.put(c,trace_increment);
        }
      }
    }
  }
  for (  Map.Entry<Production,Double> iter : data.eligibility_traces.entrySet()) {
    Production prod=iter.getKey();
    Symbol referent=prod.action_list.asMakeAction().referent.asSymbolValue().getSym();
    double temp=0.0;
    if (referent.asInteger() != null) {
      temp=referent.asInteger().getValue();
    }
 else     if (referent.asDouble() != null) {
      temp=referent.asDouble().getValue();
    }
    double delta=update * alpha * iter.getValue();
    my_agent.getTrace().print(Category.RL,"String_Node_Str",prod.getName().getValue(),temp,temp + delta);
    temp+=delta;
    prod.action_list.asMakeAction().referent=new RhsSymbolValue(my_agent.syms.createDouble(temp));
    prod.rl_update_count+=1;
    if (!prod.instantiations.isEmpty()) {
      for (      Instantiation inst : prod.instantiations) {
        for (        Preference pref : inst.preferences_generated) {
          pref.referent=my_agent.syms.createDouble(temp);
        }
      }
    }
  }
  data.reward=0.0;
  data.step=0;
  data.impasse_type=ImpasseType.NONE;
}

Fixed version (ground truth):
/** 
 * reinforcement_learning.cpp:1193:rl_perform_update
 * @param i
 * @param g
 */
public void rl_perform_update(double op_value,IdentifierImpl goal){
  ReinforcementLearningInfo data=goal.rl_info;
  boolean using_gaps=(rl_get_parameter(RL_PARAM_TEMPORAL_EXTENSION,RL_RETURN_LONG) == RL_TE_ON);
  double alpha=rl_get_parameter(RL_PARAM_LEARNING_RATE);
  double lambda=rl_get_parameter(RL_PARAM_ET_DECAY_RATE);
  double gamma=rl_get_parameter(RL_PARAM_DISCOUNT_RATE);
  double tolerance=rl_get_parameter(RL_PARAM_ET_TOLERANCE);
  double update=data.reward;
  if (using_gaps)   update*=Math.pow(gamma,(double)data.reward_age);
  update+=(Math.pow(gamma,(double)data.step) * op_value);
  update-=data.previous_q;
  rl_set_stat(RL_STAT_UPDATE_ERROR,(double)(-update));
  if (lambda == 0) {
    if (!data.eligibility_traces.isEmpty())     data.eligibility_traces.clear();
  }
 else {
    Iterator<Map.Entry<Production,Double>> it=data.eligibility_traces.entrySet().iterator();
    while (it.hasNext()) {
      Map.Entry<Production,Double> e=it.next();
      e.setValue(e.getValue() * lambda);
      e.setValue(e.getValue() * Math.pow(gamma,data.step));
      if (e.getValue() < tolerance) {
        it.remove();
      }
    }
  }
  if (data.num_prev_op_rl_rules != 0) {
    double trace_increment=(1.0 / data.num_prev_op_rl_rules);
    for (    Production c : data.prev_op_rl_rules) {
      if (c != null) {
        Double iter=data.eligibility_traces.get(c);
        if (iter != null) {
          iter+=trace_increment;
          data.eligibility_traces.put(c,iter);
        }
 else {
          data.eligibility_traces.put(c,trace_increment);
        }
      }
    }
  }
  for (  Map.Entry<Production,Double> iter : data.eligibility_traces.entrySet()) {
    Production prod=iter.getKey();
    Symbol referent=prod.action_list.asMakeAction().referent.asSymbolValue().getSym();
    double temp=0.0;
    if (referent.asInteger() != null) {
      temp=referent.asInteger().getValue();
    }
 else     if (referent.asDouble() != null) {
      temp=referent.asDouble().getValue();
    }
    double delta=update * alpha * iter.getValue();
    my_agent.getTrace().print(Category.RL,"String_Node_Str",prod.getName().getValue(),temp,temp + delta);
    temp+=delta;
    prod.action_list.asMakeAction().referent=my_agent.syms.createDouble(temp).toRhsValue();
    prod.rl_update_count+=1;
    if (!prod.instantiations.isEmpty()) {
      for (      Instantiation inst : prod.instantiations) {
        for (        Preference pref : inst.preferences_generated) {
          pref.referent=my_agent.syms.createDouble(temp);
        }
      }
    }
  }
  data.reward=0.0;
  data.step=0;
  data.impasse_type=ImpasseType.NONE;
}