Project: 003_Knowledge_Entropy_Decay_during_Language_Model_Pret | Function: turn_into_entropy (iclr)

system: You are an expert Python programmer implementing algorithms from scientific papers.
Complete the masked function based on the algorithm description.
Follow Python best practices and maintain consistent code style.
Return **only** the complete Python function code.

user: Complete this Python function:

**DESCRIPTION:** Core knowledge entropy computation function that converts MLP activation coefficients into entropy values to measure the diversity of knowledge access patterns

**KEY TERMS:** {
  "prob": "probability distribution prob(c̄ᵗᵢ)",
  "logit": "average coefficient C̄ᵗ",
  "log_probabilities": "log(prob(c̄ᵗᵢ))",
  "entropy_act_sparsity": "H(θᵗ) layer-wise entropy"
}

**CONTEXT CODE:** import argparse
import torch
import json
from collections import defaultdict
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer, OlmoForCausalLM
try:
    from .modeling_olmo_hf import ExpOlmoForCausalLM
except ImportError:
    from modeling_olmo_hf import ExpOlmoForCausalLM
import numpy as np
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader


import os
from pathlib import Path
import requests
from urllib.parse import urlparse

from olmo.config import TrainConfig
from olmo.data import build_memmap_dataset


def main(args):
    raise NotImplementedError("Function implementation has been masked for testing")
def custom_collate_fn(batch):
    # Convert the list of 'input_ids' from each sample to a tensor
    input_ids = [torch.tensor(item['input_ids']) for item in batch]
    # Stack the tensors into a batch
    input_ids = torch.stack(input_ids)
    return {'input_ids': input_ids}

def turn_into_entropy_softmax(logit):
    prob = torch.softmax(logit, dim=-1)
    log_probabilities = torch.log(prob + 1e-9)
    entropy_act_sparsity = -torch.sum(prob * log_probabilities, dim=-1)
    return entropy_act_sparsity
    
def turn_into_entropy(logit):
    raise NotImplementedError("Function implementation has been masked for testing")
def load_model(args):
            
    step = args.step
    model_size = args.model_size
    model_path = f"checkpoints/pretrained{'_1B' if model_size == '1B' else ''}/{step}-unsharded"
    base_model = "allenai/OLMo-1B-hf"
    
    tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-1B-hf")
    data_path = "data/dolma_1B/first_1k.json" 
    if not os.path.isfile(data_path):
        print("-"*50, "\nSaving decoded Dolma first batch file.")
        save_dolma()
        
    dataset = read_json_file(data_path)            
    dataset = [d['text'] for d in dataset][:args.data_size]
    print(f"\n Loaded dolma dataset \n length: {len(dataset)} \n example: {dataset[-1]}")

    instances = [d for d in range(len(dataset))]
    subset_dataset = IndexedDataset(dataset, instances, tokenizer=tokenizer) 
    dataloader = DataLoader(subset_dataset, batch_size=args.batch_size, shuffle=False) 
    
    model = ExpOlmoForCausalLM.from_pretrained(base_model, attn_implementation="eager") #, device_map="auto")  
    model = convert_to_hf(model, load_path = model_path, model_size=model_size)
        
    model.eval()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model.to(device)
    print("model_path is ", model_path)
    
    return dataloader, model, model_path

def convert_to_hf(model, load_path=None, ckpt_name=None, model_size="1B"):
    from olmo.checkpoint import load_state_dict
    from olmo.config import TrainConfig
    from olmo.util import clean_opt
    
    pt_name = "model.pt" if ckpt_name is None else f"{ckpt_name}.pt"
    ckpt = load_state_dict(
        load_path, pt_name, local_cache=None, map_location="cpu"
    )    
    
    
    yaml_path = f"configs/official/OLMo-{model_size}.yaml"
    args_list = []
    cfg = TrainConfig.load(yaml_path, [clean_opt(s) for s in args_list])
    cfg.device_eval_batch_size = 1
    
    ## CONVERT TO HF-Format
    olmo_config = cfg.model
    n_layers = olmo_config.n_layers
    n_heads = olmo_config.n_heads
    dim = olmo_config.d_model
    loaded = ckpt
    dims_per_head = dim // n_heads

    if olmo_config.n_kv_heads is not None:
        num_key_value_heads = olmo_config.n_kv_heads  # for GQA / MQA
    elif olmo_config.multi_query_attention:  # compatibility with other checkpoints
        num_key_value_heads = 1
    else:
        num_key_value_heads = n_heads

    dims_per_head = dim // n_heads
    state_dict = {}
    for layer_i in range(n_layers):
        filename = f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
        # Unsharded
        # TODO: Layernorm stuff
        # TODO: multi query attention
        fused_dims = [dim, dims_per_head * num_key_value_heads, dims_per_head * num_key_value_heads]
        q_proj_weight, k_proj_weight, v_proj_weight = torch.split(
            loaded[f"transformer.blocks.{layer_i}.att_proj.weight"], fused_dims, dim=0
        )
        up_proj_weight, gate_proj_weight = torch.chunk(
            loaded[f"transformer.blocks.{layer_i}.ff_proj.weight"], 2, dim=0
        )
        
        state_dict.update({
            f"model.layers.{layer_i}.self_attn.q_proj.weight": q_proj_weight,
            f"model.layers.{layer_i}.self_attn.k_proj.weight": k_proj_weight,
            f"model.layers.{layer_i}.self_attn.v_proj.weight": v_proj_weight,
            f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[
                f"transformer.blocks.{layer_i}.attn_out.weight"
            ],
            f"model.layers.{layer_i}.mlp.gate_proj.weight": gate_proj_weight,
            f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"transformer.blocks.{layer_i}.ff_out.weight"],
            f"model.layers.{layer_i}.mlp.up_proj.weight": up_proj_weight,
        })

            # state_dict[f"model.layers.{layer_i}.self_attn.rotary_emb.inv_freq"] = inv_freq

    state_dict.update({
        "model.embed_tokens.weight": loaded["transformer.wte.weight"],
        "lm_head.weight": loaded["transformer.ff_out.weight"]
        if "transformer.ff_out.weight" in loaded
        else loaded["transformer.wte.weight"],
    })
    ### End of Conversion.


Complete the function implementation replacing "raise NotImplementedError".