Repository: EleutherAI/lm-evaluation-harness
Issue: Evaluate a modified model
Root cause: Documentation Deficiency

How to evaluate a modified model that has not been registered as a huggingface model? I have defined a adapter called Customed class but I found it doesn't work. I'm wondering why.
class CustomLM(TemplateLM):
    def __init__(self, model, tokenizer, device="cuda", batch_size=8, max_length=1024):
        super().__init__()
        
        self._model = model
        self._tokenizer = tokenizer
        self._device = torch.device(device)
        self._model.to(self.device)
        self._model.eval()
        
        self._batch_size = batch_size
        self._max_length = max_length
        self._pad_token_id = tokenizer.pad_token_id or tokenizer.eos_token_id
        
        # 设置缓存但限制大小
        from cachetools import LRUCache
        self._cache = LRUCache(maxsize=1000)
        
    @property
    def eot_token_id(self):
        return self._tokenizer.eos_token_id
    
    @property
    def max_length(self):
        return self._max_length
    
    @property
    def max_gen_toks(self):
        return 1024
    
    @property
    def batch_size(self):
        return self._batch_size
    
    @property
    def device(self):
        return self._device
    
    @property
    def tokenizer(self):
        return self._tokenizer
    
    @property
    def model(self):
        return self._model
    
    def tok_encode(self, string: str) -> List[int]:
        return self.tokenizer.encode(string, add_special_tokens=False)
    
    def tok_decode(self, tokens: List[int]) -> str:
        return self.tokenizer.decode(tokens, skip_special_tokens=True)
    
    def _model_call(self, inps: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
        with torch.no_grad():
            if attention_mask is None:
                attention_mask = (inps != self._pad_token_id).to(self.device)
                
            # 兼容不同模型输出
            outputs = self.model(
                input_ids=inps.to(self.device),
                attention_mask=attention_mask
            )
            
            # 处理不同返回格式
            if isinstance(outputs, torch.Tensor):
                return outputs
            return outputs.logits


    def loglikelihood_rolling(self, requests: List[str]) -> List[float]:
        results = []
        stride = self.max_length // 2  # 使用50%重叠
        
        for doc in tqdm(requests, desc="Log Likelihood Rolling"):
            tokens = self.tok_encode(doc)
            total_log_prob = 0.0
            total_tokens = 0
            
            prev_end = 0
            for start in range(0, len(tokens), stride):
                end = min(start + self.max_length, len(tokens))
                
                # 跳过空窗口
                if start >= end:
                    break
                
                # 创建输入 (包含前文重叠)
                inp = tokens[max(0, start - stride):end]
                inp_tensor = torch.tensor([inp], dtype=torch.long).to(self.device)
                
                # 创建标签 (预测下一个token)
                labels = tokens[start:end]
                labels_tensor = torch.tensor([labels], dtype=torch.long).to(self.device)
                
                # 获取logits
                logits = self._model_call(inp_tensor)
                
                # 计算对数概率 (只计算当前stride部分)
                log_probs = torch.nn.functional.log_softmax(logits, dim=-1)
                
                # 计算当前窗口的有效部分
                start_idx = max(0, stride - (start - prev_end))
                end_idx = start_idx + (end - start)
                
                # 收集相关token的对数概率
                window_logits = log_probs[:, start_idx:end_idx-1]
                window_labels = labels_tensor[:, start_idx+1:end_idx]
                
                gen_probs = torch.gather(
                    window_logits, 
                    dim=-1, 
                    index=window_labels.unsqueeze(-1)
                ).squeeze(-1)
                
                # 更新总对数概率
                total_log_prob += gen_probs.sum().item()
                total_tokens += gen_probs.numel()
                prev_end = end
            
            # 平均对数概率 (perplexity计算用)
            results.append(total_log_prob / total_tokens if total_tokens > 0 else 0.0)
        
        return results

    def generate_until(self, requests: List[dict]) -> List[str]:
        results = []
        self.tokenizer.padding_side = "left"
        self.tokenizer.pad_token = self.tokenizer.pad_token or self.tokenizer.eos_token
        
        # 分批处理请求
        for i in range(0, len(requests), self.batch_size):
            batch = requests[i:i+self.batch_size]
            contexts = [req["context"] for req in batch]
            stop_sequences = [req.get("stop_sequences", []) for req in batch]
            max_gen_toks = [req.get("max_length", self.max_gen_toks) for req in batch]
            
            # 编码上下文
            inputs = self.tokenizer(
                contexts,
                padding=True,
                return_tensors="pt"
            ).to(self.device)
            
            # 创建停止条件
            stopping_criteria = []
            for stop_seq in stop_sequences:
                if stop_seq:
                    criteria = MultiTokenStoppingCriteria(
                        self.tokenizer,
                        stop_seq,
                        inputs.input_ids.shape[1]  # 上下文长度
                    )
                    stopping_criteria.append(criteria)
            
            # 批量生成
            output_ids = self.model.generate(
                inputs.input_ids,
                attention_mask=inputs.attention_mask,
                max_length=inputs.input_ids.shape[1] + max(max_gen_toks),
                stopping_criteria=StoppingCriteriaList(stopping_criteria) if stopping_criteria else None,
                pad_token_id=self._pad_token_id,
                eos_token_id=self.eot_token_id,
                do_sample=False,
            )
            
            # 解码并处理停止序列
            for j in range(len(batch)):
                # 移除输入上下文
                gen_ids = output_ids[j, inputs.input_ids.shape[1]:]
                
                # 解码
                text = self.tok_decode(gen_ids.tolist())
                
                # 应用停止序列
                for stop in stop_sequences[j]:
                    if stop in text:
                        text = text.split(stop)[0]
                
                results.append(text)
        
        return results
    
    def _loglikelihood_tokens(
        self,
        requests: List[Tuple[Tuple[str], str]]
    ) -> List[Tuple[float, bool]]:
        """
        To be continued...
        """

        results = []
        for (context_enc, continuation_enc) in tqdm(requests, desc="Log Likelihood Tokens"): 
            cache_key = (tuple(context_enc), tuple(continuation_enc))
            if cache_key in self._cache:
                results.append(self._cache[cache_key])
                continue
            inp = context_enc + continuation_enc
            inp = inp[-(self.max_length + 1):]  # 确保不超过最大长度
            inp_tensor = torch.tensor([inp], dtype=torch.long).to(self.device)
            labels = [-100] * len(context_enc) + continuation_enc
            labels = labels[-len(inp):]
            labels_tensor = torch.tensor([labels], dtype=torch.long).to(self.device)
            logits = self._model_call(inp_tensor)
            log_probs = torch.nn.functional.log_softmax(logits, dim=-1)
            shift_logits = log_probs[..., :-1, :].contiguous()
            shift_labels = labels_tensor[..., 1:].contiguous()
            loss_fct = torch.nn.CrossEntropyLoss(reduction='none')
            loss = loss_fct(
                shift_logits.view(-1, shift_logits.size(-1)),
                shift_labels.view(-1)
            ).view(shift_labels.shape)
            mask = (shift_labels != -100) & (shift_labels != self._pad_token_id)
            masked_loss = loss * mask.float()
            log_prob = masked_loss.sum(dim=-1).item() * -1
            is_greedy = True  # 默认使用贪婪解码
            self._cache[cache_key] = (log_prob, is_greedy)
            results.append((log_prob, is_greedy))
        return results
I have defined all the required function but I got the weird output： ”ask ['hellaswag'] results:" and I don't know how to deal with this.


Hi! Could you describe your error/issue a bit more?