testonline

README.md

@@ -1,13 +1,12 @@
 ---
-title: Fresh Bench
-emoji: 🏢
-colorFrom: purple
-colorTo: blue
+title: FreshBench
+emoji: 🌖
+colorFrom: blue
+colorTo: indigo
 sdk: gradio
 sdk_version: 4.21.0
 app_file: app.py
 pinned: false
-license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,202 @@
+import gradio as gr
+import os
+from transformers import AutoTokenizer
+from get_loss.get_loss_hf import run_get_loss
+import pdb
+from types import SimpleNamespace
+# os.system('git clone https://github.com/EleutherAI/lm-evaluation-harness')
+# os.system('cd lm-evaluation-harness')
+# os.system('pip install -e .')
+# -i https://pypi.tuna.tsinghua.edu.cn/simple
+# 第一个功能：基于输入文本和对应的损失值对文本进行着色展示
+
+def color_text(text_list=["hi", "FreshEval","!"], loss_list=[0.1,0.7]):
+    """
+    根据损失值为文本着色。
+    """
+    highlighted_text = []
+    loss_list=[0]+loss_list
+    for text, loss in zip(text_list, loss_list):
+        # color = "#FF0000" if float(loss) > 0.5 else "#00FF00"
+        color=loss/25
+        # highlighted_text.append({"text": text, "bg_color": color})
+        highlighted_text.append((text, color))
+
+        print('highlighted_text',highlighted_text)
+    return highlighted_text
+
+# 第二个功能：根据 ID 列表和 tokenizer 将 ID 转换为文本，并展示
+def get_text(ids_list=[0.1,0.7], tokenizer=None):
+    """
+    给定一个 ID 列表和 tokenizer 名称，将这些 ID 转换成文本。
+    """
+    # return ['Hi', 'Adam']
+    # tokenizer = AutoTokenizer.from_pretrained(tokenizer)
+    print('ids_list',ids_list)
+    # pdb.set_trace()
+    text=[]
+    for id in ids_list:
+        text.append(  tokenizer.decode(id, skip_special_tokens=True))
+    # 这里只是简单地返回文本，但是可以根据实际需求添加颜色或其他样式
+    print(f'L41:{text}')
+    return text
+
+
+# def get_ids_loss(text, tokenizer, model):
+#     """
+#     给定一个文本，model and its tokenizer,返回其对应的 IDs 和损失值。
+#     """
+#     # tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+#     # model = AutoModelForCausalLM.from_pretrained(model_name)
+#     # 这里只是简单地返回 IDs 和损失值，但是可以根据实际需求添加颜色或其他样式
+#     return [1, 2], [0.1, 0.7]
+
+
+def color_pipeline(texts=["Hi","FreshEval","!"],  model=None):
+    """
+    给定一个文本，返回其对应的着色文本。
+    """
+    print('text,model',texts,model)
+    args=SimpleNamespace(texts=texts,model=model)
+    print(f'L60,text:{texts}')
+    rtn_dic=run_get_loss(args)
+    # print(rtn_dic)
+    # pdb.set_trace()
+    # {'logit':logit,'input_ids':input_chunk,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp}
+    ids, loss =rtn_dic['input_ids'],rtn_dic['loss']#= get_ids_loss(text, tokenizer, model)
+    tokenizer=rtn_dic['tokenizer'] # get tokenizer
+    text = get_text(ids, tokenizer)
+    # print('ids, loss ,text',ids, loss ,text)
+    return color_text(text, loss)
+
+
+# TODO can this be global ? maybe need session to store info of the user
+
+# 创建 Gradio 界面
+with gr.Blocks() as demo:
+    with gr.Tab("color your text"):
+        with gr.Row():
+            text_input = gr.Textbox(label="input text", placeholder="input your text here...")
+            # TODO craw and drop the file
+
+            # loss_input = gr.Number(label="loss")
+            model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")
+            output_box=gr.HighlightedText(label="colored text")
+            # gr.Examples(
+            #     [
+            #         # ["Hi FreshEval !", "microsoft/phi-2"],
+            #         ["Hello FreshBench !", "/home/sribd/chenghao/models/phi-2"],
+            #     ],
+            #     [text_input, model_input],
+            #     cache_examples=True,
+            #     # cache_examples=False,
+            #     fn=color_pipeline,
+            #     outputs=output_box
+            # )
+            # TODO select models that can be used online
+            # TODO maybe add our own models
+
+
+            color_text_output = gr.HTML(label="colored text")
+
+        color_text_button = gr.Button("color the text").click(color_pipeline, inputs=[text_input, model_input], outputs=output_box)
+
+
+        date_time_input = gr.Textbox(label="the date when the text is generated")#TODO add date time input
+        description_input = gr.Textbox(label="description of the text")
+        submit_button = gr.Button("submit a post or record").click()
+        #TODO add model and its score
+
+    with gr.Tab('test your qeustion'):
+        '''
+        use extract, or use ppl
+        '''
+        question=gr.Textbox(placeholder='input your question here...')
+        answer=gr.Textbox(placeholder='input your answer here...')
+        other_choices=gr.Textbox(placeholder='input your other choices here...')
+        
+        test_button=gr.Button('test').click()
+        #TODO add the model and its score
+
+        def test_question(question, answer, other_choices):
+            '''
+            use extract, or use ppl
+            '''
+            answer_ppl, other_choices_ppl = get_ppl(question, answer, other_choices)
+            return answer_ppl, other_choices_ppl
+
+
+
+    with gr.Tab("model text ppl with time"):
+        '''
+        see the matplotlib example, to see ppl with time, select the models
+        '''
+        # load the json file with time,
+
+    
+    with gr.Tab("model quesion acc with time"):
+        '''
+        see the matplotlib example, to see ppl with time, select the models
+        ''' 
+        #
+
+
+    with gr.Tab("hot questions"):
+        '''
+        see the questions and answers
+        '''
+        with gr.Tab("ppl"):
+            '''
+            see the questions
+            '''
+        
+
+demo.launch(debug=True)
+
+
+
+
+
+# import gradio as gr
+# import os
+# os.system('python -m spacy download en_core_web_sm')
+# import spacy
+# from spacy import displacy
+
+# nlp = spacy.load("en_core_web_sm")
+
+# def text_analysis(text):
+#     doc = nlp(text)
+#     html = displacy.render(doc, style="dep", page=True)
+#     html = (
+#         "<div style='max-width:100%; max-height:360px; overflow:auto'>"
+#         + html
+#         + "</div>"
+#     )
+#     pos_count = {
+#         "char_count": len(text),
+#         "token_count": 0,
+#     }
+#     pos_tokens = []
+
+#     for token in doc:
+#         pos_tokens.extend([(token.text, token.pos_), (" ", None)])
+
+#     return pos_tokens, pos_count, html
+
+# demo = gr.Interface(
+#     text_analysis,
+#     gr.Textbox(placeholder="Enter sentence here..."),
+#     ["highlight", "json", "html"],
+#     examples=[
+#         ["What a beautiful morning for a walk!"],
+#         ["It was the best of times, it was the worst of times."],
+#     ],
+# )
+
+# demo.launch()
+
+
+
+# # lm-eval 
+# # lm-evaluation-harness 

diff_color.py

@@ -0,0 +1,42 @@
+
+from difflib import Differ
+
+import gradio as gr
+
+
+def diff_texts(text1, text2):
+    d = Differ()
+    rtn =[
+        (token[2:], token[0] if token[0] != " " else None)
+        for token in d.compare(text1, text2)
+    ]
+    print(rtn)
+    return rtn
+
+
+demo = gr.Interface(
+    diff_texts,
+    [
+        gr.Textbox(
+            label="Text 1",
+            info="Initial text",
+            lines=3,
+            value="The quick brown fox jumped over the lazy dogs.",
+        ),
+        gr.Textbox(
+            label="Text 2",
+            info="Text to compare",
+            lines=3,
+            value="The fast brown fox jumps over lazy dogs.",
+        ),
+    ],
+    gr.HighlightedText(
+        label="Diff",
+        combine_adjacent=True,
+        show_legend=True,
+        color_map={"+": "red", "-": "green"}),
+    theme=gr.themes.Base()# the return is here
+)
+if __name__ == "__main__":
+    demo.launch()
+

flagged/log.csv

@@ -0,0 +1,2 @@
+Text 1,Text 2,Diff,flag,username,timestamp
+The quick brown fox jumped over the lazy dogs.,The fast brown fox jumps over lazy dogs.,,,,2024-03-13 16:50:01.853095

get_loss/get_loss.py

@@ -0,0 +1,297 @@
+# import packages
+import os
+from tqdm import tqdm
+import warnings
+import json
+import torch.nn.functional as F
+import torch
+import gc
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from datetime import datetime
+import argparse
+import mamba_ssm
+import rwkv
+
+
+RWKV4_TOKENIZER_FILE = "./support/20B_tokenizer.json"
+
+
+def load_list_from_json(file_path):
+    """
+    Loads a list of strings from a JSON file.
+
+    :param file_path: Path of the JSON file to be loaded.
+    :return: List of strings loaded from the JSON file.
+    """
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return json.load(file)
+
+
+def calculate_log_sum(logits, target_token_ids):
+    shifted_logits = logits[:-1, :]
+    shifted_targets = target_token_ids[1:]
+
+    log_probs = F.log_softmax(shifted_logits, dim=-1)
+
+    target_log_probs = -log_probs.gather(1, shifted_targets.unsqueeze(1)).squeeze()
+    # print(target_log_probs)
+
+    log_sum = torch.sum(target_log_probs, dim=-1)
+    # print(perplexity_sum)
+
+    return log_sum.item()
+
+
+def print_model_parameters_in_billions(model):
+    total_params = sum(p.numel() for p in model.parameters())
+
+    total_params_billion = total_params / 1e9
+
+    print(f"Model parameters: {total_params_billion:.3f} billion")
+
+
+def make_log(data_dict, folder_path):
+    if not os.path.exists(folder_path):
+        try:
+            os.makedirs(folder_path)
+            print(f"Directory created at {folder_path}")
+        except Exception as e:
+            print(f"Error creating directory: {e}")
+            return
+
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    file_name = f"{timestamp}.json"
+    file_path = os.path.join(folder_path, file_name)
+
+    try:
+        with open(file_path, 'w') as file:
+            json.dump(data_dict, file, indent=4)
+        print(f"Dictionary saved successfully to {file_path}")
+    except Exception as e:
+        print(f"Error saving dictionary: {e}")
+
+
+def load_rwkv(path):
+    os.environ['RWKV_JIT_ON'] = '1'
+    os.environ["RWKV_CUDA_ON"] = '1'
+
+    from rwkv.model import RWKV
+    from rwkv.utils import PIPELINE
+
+    rwkv_model = RWKV(model=path, strategy='cuda fp16')
+    rwkv_pipeline = PIPELINE(rwkv_model, r"rwkv_vocab_v20230424")
+    rwkv_tokenizer = rwkv_pipeline.tokenizer
+
+    return rwkv_model, rwkv_tokenizer
+
+
+def load_rwkv4pile(path):
+    os.environ['RWKV_JIT_ON'] = '1'
+    os.environ["RWKV_CUDA_ON"] = '1'
+
+    from rwkv.model import RWKV
+    from rwkv.utils import PIPELINE
+
+    rwkv_model = RWKV(model=path, strategy='cuda fp16')
+    rwkv_pipeline = PIPELINE(rwkv_model, RWKV4_TOKENIZER_FILE)
+    rwkv_tokenizer = rwkv_pipeline.tokenizer
+
+    return rwkv_model, rwkv_tokenizer
+
+
+def load_hf_model(path, cache_path):
+    hf_tokenizer = AutoTokenizer.from_pretrained(path)
+    if cache_path is not None:
+        hf_model = AutoModelForCausalLM.from_pretrained(path,
+                                                        device_map="cuda",
+                                                        trust_remote_code=True,
+                                                        cache_dir=cache_path).eval()
+    else:
+        hf_model = AutoModelForCausalLM.from_pretrained(path,
+                                                        device_map="cuda",
+                                                        trust_remote_code=True).eval()
+
+    print_model_parameters_in_billions(hf_model)
+
+    return hf_model, hf_tokenizer
+
+
+def load_mamba(path):
+    from mamba_ssm.models.mixer_seq_simple import MambaLMHeadModel
+
+    mamba_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+    mamba_model = MambaLMHeadModel.from_pretrained(path, device="cuda", dtype=torch.float16)
+    mamba_model.device = torch.device('cuda')
+
+    print_model_parameters_in_billions(mamba_model)
+
+    return mamba_model, mamba_tokenizer
+
+
+def eval_rwkv(model, tokenizer, texts, chunk_size, v4pile=False):
+    rwkv_test_data = []
+    rwkv_token_length_list = []
+
+    for idx, sample in tqdm(enumerate(texts), total=len(texts)):
+
+        with torch.no_grad():
+
+            if v4pile:
+                input_seq = tokenizer.encode(sample).ids  # v4
+            else:
+                input_seq = tokenizer.encode(sample)
+
+            input_length = len(input_seq)
+
+            neg_log_prob_temp = 0
+            # for begin in range(0, input_length, chunk_size):
+            input_chunk = input_seq[:chunk_size]
+
+            logit = model.forward(input_chunk, None, full_output=True)[0]
+
+            if len(input_chunk) == 1:
+                logit = logit.unsqueeze(0)
+
+    #             log_sum = calculate_log_sum(logit, torch.tensor(input_chunk).cuda())
+
+    #             neg_log_prob_temp += log_sum
+
+    #         rwkv_token_length_list.append(input_length)
+    #         rwkv_test_data.append(neg_log_prob_temp)
+
+    # data_dict = {
+    #     'neg_log_prob_sum': sum(rwkv_test_data) / len(rwkv_test_data),
+    #     'avg tokens': sum(rwkv_token_length_list) / len(rwkv_token_length_list),
+    # }
+
+    # print(f'log probability sum: {sum(rwkv_test_data) / len(rwkv_test_data):.2f}')
+    # print(f'avg tokens: {sum(rwkv_token_length_list) / len(rwkv_token_length_list):.0f}')
+
+    return logit,logit,input_chunk,tokenizer
+
+
+def eval_hf_model(model, tokenizer, texts, chunk_size):
+    data = []
+    token_length_list = []
+
+    for idx, sample in tqdm(enumerate(texts), total=len(texts)):
+
+        with torch.no_grad():
+
+            inputs = tokenizer(sample, return_tensors='pt')
+            inputs = inputs.to(model.device)
+
+            seq_length = inputs['input_ids'].shape[-1]
+
+            neg_log_prob_temp = 0
+            # for begin in range(0, seq_length, chunk_size):
+            input_chunk = inputs['input_ids'][:, :chunk_size]
+
+            logit = model.forward(input_ids=input_chunk).logits[0, :, :]
+
+    #             log_sum = calculate_log_sum(logit, input_chunk.squeeze(0))
+    #             neg_log_prob_temp += log_sum
+
+    #         token_length_list.append(seq_length)
+    #         data.append(neg_log_prob_temp)
+
+    # data_dict = {
+    #     'neg_log_prob_sum': sum(data) / len(data),
+    #     'avg tokens': sum(token_length_list) / len(token_length_list),
+    # }
+
+    # print(f'log probability sum: {sum(data) / len(data):.2f}')
+    # print(f'avg tokens: {sum(token_length_list) / len(token_length_list):.0f}')
+
+    return logit,input_chunk,tokenizer
+
+
+# if __name__ == '__main__':
+#     parser = argparse.ArgumentParser()
+
+#     parser.add_argument('--model', type=str, required=True, help='model name or path')
+#     parser.add_argument('--model_type', choices=['hf', 'rwkv', 'mamba', 'rwkv4pile'], required=True, help='model type')
+#     parser.add_argument('--data', type=str, required=True, help='data path (json file)')
+#     parser.add_argument('--log_path', type=str, default='./logs/', help='log file path')
+#     parser.add_argument('--model_cache', type=str, help='hugging face model cache')
+#     parser.add_argument('--chunk_size', type=int, default=1024, help='chunk size')
+
+
+def run_get_loss(args):
+    # args = parser.parse_args()
+
+    # load data
+    texts = load_list_from_json(args.data)
+    print(f'data size: {len(texts)}')
+
+    # load model
+    if args.model_type == 'hf':
+        model, tokenizer = load_hf_model(args.model, args.model_cache)# tokenzier path, model path
+    elif args.model_type == 'rwkv':
+        model, tokenizer = load_rwkv(args.model)
+    elif args.model_type == 'mamba':
+        model, tokenizer = load_mamba(args.model)
+    elif args.model_type == 'rwkv4pile':
+        model, tokenizer = load_rwkv4pile(args.model)
+    else:
+        raise NotImplementedError
+
+    # eval
+    if args.model_type in ['hf', 'mamba']:
+        return eval_hf_model(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
+    elif args.model_type == 'rwkv':
+        return eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
+    elif args.model_type == 'rwkv4pile':
+        return eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size, v4pile=True)
+    else:
+        raise NotImplementedError
+
+    # results['model_name_or_path'] = args.model
+    # results['data_path'] = args.data
+    # results['chunk_size'] = args.chunk_size
+
+    # make_log(results, args.log_path)
+
+    # print(json.dumps(results, indent=4, ensure_ascii=False))
+
+from types import SimpleNamespace
+
+if __name__ == '__main__':
+    args=SimpleNamespace(model='microsoft/phi-2',texts=['Hello FreshBench !'],model_type='hf',data='data.json',model_cache=None,chunk_size=1024)
+
+
+
+# def run_get_loss(input_string, model_type):
+#     # load data
+#     texts = [input_string]
+#     print(f'data size: {len(texts)}')
+
+#     # load model
+#     if model_type == 'hf':
+#         model, tokenizer = load_hf_model(args.model, args.model_cache)# tokenzier path, model path
+#     elif model_type == 'rwkv':
+#         model, tokenizer = load_rwkv(args.model)
+#     elif model_type == 'mamba':
+#         model, tokenizer = load_mamba(args.model)
+#     elif model_type == 'rwkv4pile':
+#         model, tokenizer = load_rwkv4pile(args.model)
+#     else:
+#         raise NotImplementedError
+
+#     # eval
+#     if model_type in ['hf', 'mamba']:
+#         results = eval_hf_model(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
+#     elif model_type == 'rwkv':
+#         results = eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
+#     elif model_type == 'rwkv4pile':
+#         results = eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size, v4pile=True)
+#     else:
+#         raise NotImplementedError
+
+#     results['model_name_or_path'] = args.model
+#     results['data_path'] = args.data
+#     results['chunk_size'] = args.chunk_size
+
+#     make_log(results, args.log_path)
+
+#     print(json.dumps(results, indent=4, ensure_ascii=False))

get_loss/get_loss_hf.py

@@ -0,0 +1,306 @@
+# import packages
+import os
+# from tqdm import tqdm
+# import warnings
+import json
+import torch.nn.functional as F
+import torch
+import gc
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from datetime import datetime
+import argparse
+from types import SimpleNamespace
+import pdb
+# import mamba_ssm
+# import rwkv
+
+
+# RWKV4_TOKENIZER_FILE = "./support/20B_tokenizer.json"
+# device = 'cuda' if torch.cuda.is_available() else 'cpu'
+device = 'cpu'
+
+
+def load_list_from_json(file_path):
+    """
+    Loads a list of strings from a JSON file.
+
+    :param file_path: Path of the JSON file to be loaded.
+    :return: List of strings loaded from the JSON file.
+    """
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return json.load(file)
+
+
+def calculate_loss(logits, target_token_ids):
+    # shifted_logits = logits[:-1, :]
+    # shifted_targets = target_token_ids[1:]
+
+    # log_probs = F.log_softmax(shifted_logits, dim=-1)
+    loss = torch.nn.functional.cross_entropy(logits[:-1, :].view(-1, logits.shape[-1]), 
+                                             target_token_ids[1:].view(-1), reduction='none')
+    # pdb.set_trace()
+
+
+    # target_log_probs = -log_probs.gather(1, shifted_targets.unsqueeze(1)).squeeze()
+    # # print(target_log_probs)
+
+    # log_sum = torch.sum(target_log_probs, dim=-1)
+    # print(perplexity_sum)
+
+    return loss.cpu().numpy()
+
+
+def calculate_log_sum(logits, target_token_ids):
+    shifted_logits = logits[:-1, :]
+    shifted_targets = target_token_ids[1:]
+
+    log_probs = F.log_softmax(shifted_logits, dim=-1)
+
+    target_log_probs = -log_probs.gather(1, shifted_targets.unsqueeze(1)).squeeze()
+    # print(target_log_probs)
+
+    log_sum = torch.sum(target_log_probs, dim=-1)
+    # print(perplexity_sum)
+
+    return log_sum.item()
+
+
+def print_model_parameters_in_billions(model):
+    total_params = sum(p.numel() for p in model.parameters())
+
+    total_params_billion = total_params / 1e9
+
+    print(f"Model parameters: {total_params_billion:.3f} billion")
+
+
+# def make_log(data_dict, folder_path):
+#     if not os.path.exists(folder_path):
+#         try:
+#             os.makedirs(folder_path)
+#             print(f"Directory created at {folder_path}")
+#         except Exception as e:
+#             print(f"Error creating directory: {e}")
+#             return
+
+#     timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+#     file_name = f"{timestamp}.json"
+#     file_path = os.path.join(folder_path, file_name)
+
+#     try:
+#         with open(file_path, 'w') as file:
+#             json.dump(data_dict, file, indent=4)
+#         print(f"Dictionary saved successfully to {file_path}")
+#     except Exception as e:
+#         print(f"Error saving dictionary: {e}")
+
+
+# def load_rwkv(path):
+#     os.environ['RWKV_JIT_ON'] = '1'
+#     os.environ["RWKV_CUDA_ON"] = '1'
+
+#     from rwkv.model import RWKV
+#     from rwkv.utils import PIPELINE
+
+#     rwkv_model = RWKV(model=path, strategy='cuda fp16')
+#     rwkv_pipeline = PIPELINE(rwkv_model, r"rwkv_vocab_v20230424")
+#     rwkv_tokenizer = rwkv_pipeline.tokenizer
+
+#     return rwkv_model, rwkv_tokenizer
+
+
+# def load_rwkv4pile(path):
+#     os.environ['RWKV_JIT_ON'] = '1'
+#     os.environ["RWKV_CUDA_ON"] = '1'
+
+#     from rwkv.model import RWKV
+#     from rwkv.utils import PIPELINE
+
+#     rwkv_model = RWKV(model=path, strategy='cuda fp16')
+#     rwkv_pipeline = PIPELINE(rwkv_model, RWKV4_TOKENIZER_FILE)
+#     rwkv_tokenizer = rwkv_pipeline.tokenizer
+
+#     return rwkv_model, rwkv_tokenizer
+
+
+def load_hf_model(path, cache_path):
+    hf_tokenizer = AutoTokenizer.from_pretrained(path)
+    if cache_path is not None:
+        # pdb.set_trace()
+        hf_model = AutoModelForCausalLM.from_pretrained(path,
+                                                        device_map=device,
+                                                        trust_remote_code=True,
+                                                        cache_dir=cache_path).eval()
+    else:
+        hf_model = AutoModelForCausalLM.from_pretrained(path,
+                                                        device_map=device,
+                                                        trust_remote_code=True).eval()
+
+    print_model_parameters_in_billions(hf_model)
+
+    return hf_model, hf_tokenizer
+
+
+# def load_mamba(path):
+#     from mamba_ssm.models.mixer_seq_simple import MambaLMHeadModel
+
+#     mamba_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+#     mamba_model = MambaLMHeadModel.from_pretrained(path, device="cuda", dtype=torch.float16)
+#     mamba_model.device = torch.device('cuda')
+
+#     print_model_parameters_in_billions(mamba_model)
+
+#     return mamba_model, mamba_tokenizer
+
+
+# def eval_rwkv(model, tokenizer, texts, chunk_size, v4pile=False):
+#     rwkv_test_data = []
+#     rwkv_token_length_list = []
+
+#     for idx, sample in tqdm(enumerate(texts), total=len(texts)):
+
+#         with torch.no_grad():
+
+#             if v4pile:
+#                 input_seq = tokenizer.encode(sample).ids  # v4
+#             else:
+#                 input_seq = tokenizer.encode(sample)
+
+#             input_length = len(input_seq)
+
+#             neg_log_prob_temp = 0
+#             # for begin in range(0, input_length, chunk_size):
+#             input_chunk = input_seq[:chunk_size]
+
+#             logit = model.forward(input_chunk, None, full_output=True)[0]
+
+#             if len(input_chunk) == 1:
+#                 logit = logit.unsqueeze(0)
+
+    #             log_sum = calculate_log_sum(logit, torch.tensor(input_chunk).cuda())
+
+    #             neg_log_prob_temp += log_sum
+
+    #         rwkv_token_length_list.append(input_length)
+    #         rwkv_test_data.append(neg_log_prob_temp)
+
+    # data_dict = {
+    #     'neg_log_prob_sum': sum(rwkv_test_data) / len(rwkv_test_data),
+    #     'avg tokens': sum(rwkv_token_length_list) / len(rwkv_token_length_list),
+    # }
+
+    # print(f'log probability sum: {sum(rwkv_test_data) / len(rwkv_test_data):.2f}')
+    # print(f'avg tokens: {sum(rwkv_token_length_list) / len(rwkv_token_length_list):.0f}')
+
+    return logit,logit,input_chunk,tokenizer
+
+
+def eval_hf_model(model, tokenizer, texts, chunk_size):
+    data = []
+    token_length_list = []
+
+    # for idx, sample in tqdm(enumerate(texts), total=len(texts)):#TODO deleta the forloop
+    with torch.no_grad():
+
+        inputs = tokenizer(texts, return_tensors='pt')
+        inputs = inputs.to(model.device)
+
+        seq_length = inputs['input_ids'].shape[-1]
+
+        neg_log_prob_temp = 0
+        # for begin in range(0, seq_length, chunk_size):
+        input_chunk = inputs['input_ids'][:, :chunk_size]
+
+        logit = model.forward(input_ids=input_chunk).logits[0, :, :]
+
+        log_sum = calculate_log_sum(logit, input_chunk.squeeze(0))# suppose shape of logit is (seq_length, vocab_size),shape of input_chunk is (,seq_length)
+        neg_log_prob_temp += log_sum
+
+        loss = calculate_loss(logit, input_chunk.squeeze(0))
+
+    #         token_length_list.append(seq_length)
+    #         data.append(neg_log_prob_temp)
+
+    # data_dict = {
+    #     'neg_log_prob_sum': sum(data) / len(data),
+    #     'avg tokens': sum(token_length_list) / len(token_length_list),
+    # }
+
+    # print(f'log probability sum: {sum(data) / len(data):.2f}')
+    # print(f'avg tokens: {sum(token_length_list) / len(token_length_list):.0f}')
+        rtn_dic={'logit':logit.cpu().numpy(),'input_ids':input_chunk.cpu().numpy()[0],'loss':loss,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp}
+        return rtn_dic
+
+
+# if __name__ == '__main__':
+#     parser = argparse.ArgumentParser()
+
+#     parser.add_argument('--model', type=str, required=True, help='model name or path')
+#     parser.add_argument('--model_type', choices=['hf', 'rwkv', 'mamba', 'rwkv4pile'], required=True, help='model type')
+#     parser.add_argument('--data', type=str, required=True, help='data path (json file)')
+#     parser.add_argument('--log_path', type=str, default='./logs/', help='log file path')
+#     parser.add_argument('--model_cache', type=str, help='hugging face model cache')
+#     parser.add_argument('--chunk_size', type=int, default=1024, help='chunk size')
+
+
+def run_get_loss(args=None):
+    if args is None:
+        # args=SimpleNamespace(model='microsoft/phi-2',texts='Hello FreshBench !',model_type='hf',model_cache=None,chunk_size=1024)
+        args=SimpleNamespace(model='/home/sribd/chenghao/models/phi-2',texts='Hello FreshBench !',model_type='hf',model_cache=None,chunk_size=1024)
+
+    
+    if 'chunk_size' not in args.__dict__:
+        args.chunk_size=1024
+    if 'model_type' not in args.__dict__:
+        args.model_type='hf'
+    if 'model' not in args.__dict__ or len(args.model)<2:
+        # args.model='/home/sribd/chenghao/models/phi-2'
+        args.model='microsoft/phi-2'
+
+    if 'model_cache' not in args.__dict__:
+        args.model_cache=args.model
+
+    # args = parser.parse_args()
+
+    # load data
+    # texts = load_list_from_json(args.data)
+    print('args',args)
+    texts=args.texts
+    print(f'data size: {len(texts)}')
+
+    # load model
+    if args.model_type == 'hf':
+        model, tokenizer = load_hf_model(args.model, args.model_cache)# tokenzier path, model path
+    # elif args.model_type == 'rwkv':
+    #     model, tokenizer = load_rwkv(args.model)
+    # elif args.model_type == 'mamba':
+    #     model, tokenizer = load_mamba(args.model)
+    # elif args.model_type == 'rwkv4pile':
+    #     model, tokenizer = load_rwkv4pile(args.model)
+    else:
+        raise NotImplementedError
+
+    # eval
+    if args.model_type in ['hf', 'mamba']:
+        print(f'eval hf')
+        return eval_hf_model(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
+    # elif args.model_type == 'rwkv':
+    #     return eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
+    # elif args.model_type == 'rwkv4pile':
+    #     return eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size, v4pile=True)
+    else:
+        raise NotImplementedError
+
+    # results['model_name_or_path'] = args.model
+    # results['data_path'] = args.data
+    # results['chunk_size'] = args.chunk_size
+
+    # make_log(results, args.log_path)
+
+    # print(json.dumps(results, indent=4, ensure_ascii=False))
+
+
+if __name__ == '__main__':
+    args=SimpleNamespace(model='microsoft/phi-2',texts='Hello FreshBench !',model_type='hf',model_cache=None,chunk_size=1024)
+    run_get_loss(args)
+
+# run_get_loss(args)

get_loss/my_get_logit.py

@@ -0,0 +1,334 @@
+
+import logging
+import warnings
+import os
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+import transformers
+import torch
+import gc
+from torch.utils.data import DataLoader, TensorDataset
+from torch.nn.utils.rnn import pack_padded_sequence
+
+
+from calc_metrics import calculate_log_sum,calculate_log_last
+import torch.nn.functional as F
+import logging
+import time
+import traceback
+
+import datetime
+doday=datetime.datetime.now().strftime("%Y-%m-%d")
+# 配置日志
+extra_info='fill'
+
+# logging.basicConfig(level=logging.INFO,filename='/wangbenyou/chenghao/fersh_bench/log/app.log', filemode='a', format='%(name)s - %(levelname)s - %(message)s')
+# logging.basicConfig(level=logging.INFO,filename=f'../log/app_jieduan_{extra_info}{doday}_year.log', filemode='a', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+import torch
+import pdb
+import json
+
+
+paths=[
+'/mntcephfs/data/med/fanyaxin/Qwen-7B-Chat',
+
+] 
+
+
+
+# file_in_data_folder='2024-01-04_18'
+# file_in_data_folder='2023-12-31'
+file_in_data_folder='2023-12-27'
+# file_in_data_folder='2020_100'
+# file_in_data_folder='2020'
+# file_in_data_folder='2014'
+# file_in_data_folder='2017'
+# file_in_data_folder='2019'
+# file_in_data_folder='2019'
+# file_in_data_folder='rephrase_MMLU'
+# file_in_data_folder='mock_MMLU'
+
+# mmlu_mock_concat
+
+# not arxiv not year, but rep MMLU
+# 你的语料列表
+import get_text
+# file_dic_list_strings=get_text.file_dic_list_strings
+limit_lines_per_file=10
+file_dic_list_strings=get_text.get_text_from(file_in_data_folder,limit=limit_lines_per_file)
+# file_dic_list_strings=get_text.get_mmlu_rephrase_text(directory='/mntnfs/med_data5/chenghao/fresh_eval/data/mmlu_rephrase_concat/gpt-4-1106-preview/')
+# file_dic_list_strings=get_text.get_mmlu_rephrase_text(directory='/mntnfs/med_data5/chenghao/fresh_eval/data/mmlu_mock_concat/gpt-4-1106-preview/')
+        
+
+
+# file_in_data_folder='2024-01-03'
+
+def get_rwkv_model_tokenizer(model_name):
+    os.environ['RWKV_JIT_ON'] = '1'
+    os.environ["RWKV_CUDA_ON"] = '1'
+    from rwkv.model import RWKV
+    from rwkv.utils import PIPELINE
+    model=RWKV(model=model_name, strategy='cuda fp16')
+    pipeline = PIPELINE(model, r"rwkv_vocab_v20230424")
+    tokenizer = pipeline.tokenizer
+    return model,tokenizer
+
+def get_mamba_model_tokenizer(model_name):
+    from mamba_ssm.models.mixer_seq_simple import MambaLMHeadModel
+    device = "cuda"
+    tokenizer = AutoTokenizer.from_pretrained("/mntcephfs/data/med/chenghao/models/gpt-neox-20b_tokenizer")
+    model = MambaLMHeadModel.from_pretrained(model_name, device=device, dtype=torch.float16)
+    return model,tokenizer
+
+
+def get_HF_model_tokenizer(model_name):
+    if 'llama_hf_13b' in model_name:
+        tokenizer = transformers.LlamaTokenizer.from_pretrained(model_name, unk_token="<unk>") 
+    else:
+        from transformers import AutoTokenizer, AutoModelForCausalLM
+
+        tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+    
+    if 'zephyr' in model_name.lower():
+        model = AutoModelForCausalLM.from_pretrained(model_name,device_map="auto").eval()
+    
+    else:
+        model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", trust_remote_code=True).eval()
+    return model,tokenizer
+
+limit_lines_per_file=10
+
+def run_model_on_dic(config):
+    config['clear_log_first']=True
+    logging.info("start up")
+    paths=config['model_path']
+    file_dic_list_strings=config['file_dic_list_strings']
+    detail_log_base=config['detail_log_path']
+    extract_log_base=config['extract_log_path']
+    max_sequence_length,max_str_len,limit_lines_per_file=config['max_sequence_length'],config['max_str_len'],config['limit_lines_per_file']
+    
+    for model_name in tqdm(paths):
+        model_name=model_name.strip()
+        tmp_path=model_name[:-1] if model_name[-1]=='/' else model_name
+        short_model_name=tmp_path.split('/')[-1]
+        config['detail_log_path']=detail_log_base.replace('TOFILL',f'{short_model_name}')
+        config['extract_log_path']=extract_log_base.replace('TOFILL',f'{short_model_name}')
+        if 'clear_log_first' in config.keys() and config['clear_log_first'] is True:
+            with open( config['extract_log_path'],'w')as f:
+                f.write('')
+            with open( config['detail_log_path'],'w')as f:
+                f.write('')
+            print(f'\n log cleared! ')
+
+        logging.basicConfig(level=logging.INFO,filename=config['detail_log_path'], filemode='a', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',force=True)
+
+        
+
+        print()
+        print('model_path',model_name)
+        print(f'extract_log_path:{config["extract_log_path"]}\ndetail_log_path:{config["detail_log_path"]}')
+        print()
+
+        try:
+            if config['model_type']=='RWKV':#'HF' not in model_name and (('RWKV' in model_name) or ('rwkv' in model_name )):
+                model,tokenizer=get_rwkv_model_tokenizer(model_name)
+
+
+            elif config['model_type']=='MAMBA':#('mamba' in model_name) or ('MAMBA'in model_name ):
+                model,tokenizer=get_mamba_model_tokenizer(model_name)
+
+
+            elif config['model_type']=='HF':#'HF' in model_name:
+
+                model,tokenizer=get_HF_model_tokenizer(model_name)
+                print(f'model device:{model.device}')
+                print('[tokenizer.cls_token]',[tokenizer.cls_token])
+                print('[tokenizer.sep_token]',[tokenizer.sep_token])
+            else:
+                raise Exception('model type not found')
+
+            # === get model and tokenizer
+                
+            for file_name,corpus in file_dic_list_strings.items():
+
+                tokenized_corpus=[]
+                for text in corpus:
+                    text=text[:max_str_len]
+                    if config['model_type']=='RWKV':
+                              #'HF' not in model_name and (('RWKV' in model_name) or ('rwkv' in model_name )):
+                        tokenized_corpus.append(tokenizer.encode(text))
+
+                    elif 'HF' in model_name and ('RWKV' in model_name):
+                        tokenized_corpus.append(tokenizer(text, return_tensors="pt")['input_ids'])
+
+                    elif ('mamba' in model_name) or ('MAMBA'in model_name ):
+                        device=torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+                        tokenized_corpus.append(tokenizer(text, return_tensors="pt").input_ids.to(device=device))
+
+                    else:
+                        tokens = tokenizer.tokenize(text)
+                        if tokenizer.cls_token:# attention here is not [None]
+                            tokens = [tokenizer.cls_token] + tokens 
+                        if tokenizer.sep_token:
+                            tokens = tokens +[tokenizer.sep_token]
+                        input_ids = tokenizer.convert_tokens_to_ids(tokens)
+                        tokenized_corpus.append(input_ids)
+                        # tokenized_corpus.append(tokenizer(text, return_tensors="pt")['input_ids'])
+
+                
+
+                processed_sequences = []
+
+                # 遍历 tokenized_corpus，截断或补全序列
+                for sequence in tokenized_corpus:
+                    # print('len(sequence)',len(sequence))
+                    if len(sequence) < max_sequence_length:
+                        pass
+                        # 补全序列
+                        # sequence = sequence + [tokenizer.pad_token_id] * (max_sequence_length - len(sequence))
+                        # print(f'longer {max_sequence_length - len(sequence)}')
+                    elif len(sequence) > max_sequence_length:
+                        # 截断序列
+                        sequence = sequence[:max_sequence_length]
+                    
+                    # 将处理后的序列添加到列表中
+                    processed_sequences.append(sequence)
+
+
+                total_loss = 0.0
+                total_tokens = 0
+                # pdb.set_trace()
+
+                for enu,batch_input_ids in tqdm(enumerate(processed_sequences)):
+                    # if 'test_fun_dev' in config['detail_log_path'] and enu>50:
+                    #     print(f'enu:{enu} batch_input_ids: break')
+                    #     break
+
+                    batch_input_ids=torch.tensor(batch_input_ids).unsqueeze(0)
+
+                    with torch.no_grad():
+                        # 获取模型的输出
+                        # pdb.set_trace()
+                        if config['model_type']=='RWKV':
+                        # if 'HF' not in model_name and (('RWKV' in model_name) or ('rwkv' in model_name )):
+                            # print('rwkv1')
+                            # pdb.set_trace()
+                            # logits = model.forward(batch_input_ids.squeeze().to(torch.float32), None, full_output=True)[0]
+                            logits = model.forward(batch_input_ids.squeeze().long(), None, full_output=True)[0]
+                            # logits = model.forward(batch_input_ids.squeeze(), None, full_output=True)[0]
+                            # print(logits.shape)   
+                            '''
+                            tmp=torch.tensor(batch_input_ids).unsqueeze(0)
+                            logits = model.forward(batch_input_ids.squeeze().long(), None)
+                            logits = model.forward(batch_input_ids.long(), None,)[0]
+                            for output in outputs:print(tokenizer.decode(output.tolist(), skip_special_tokens=True))
+
+                            '''
+                            # loss = torch.nn.functional.cross_entropy(logits[ :-1, :].view(-1, logits.shape[-1]).to(torch.float32), batch_input_ids[0,1:].to(logits.device).view(-1).to(torch.float32), reduction='none')
+                            loss = torch.nn.functional.cross_entropy(logits[ :-1, :].view(-1, logits.shape[-1]).to(torch.float32), batch_input_ids[0,1:].to(logits.device).view(-1), reduction='none')
+
+                        elif config['model_type']=='MAMBA':
+                            # pdb.set_trace() 
+                            mamba_output = model.forward(batch_input_ids[0])#the shape should be like (1,length)
+                            logits = mamba_output.logits
+                            loss = torch.nn.functional.cross_entropy(logits[:, :-1, :].view(-1, logits.shape[-1]), batch_input_ids[0][:,1:].view(-1), reduction='none')
+                            # pdb.set_trace()
+
+
+
+                        elif config['model_type']=='HF':
+                            if 'HF' in model_name and 'RWKV' in model_name:
+                                # pdb.set_trace()
+                                batch_input_ids=batch_input_ids.to(model.device)
+                                logits = model.forward(batch_input_ids[0]).logits#the shape should be like (1,length)
+                                loss = torch.nn.functional.cross_entropy(logits[:, :-1, :].view(-1, logits.shape[-1]), batch_input_ids[0][:,1:].view(-1), reduction='none')
+                                '''
+                                batch_input_ids=batch_input_ids.to(model.device)
+
+    HuggingFace-Download-Accelerator/
+    (Pdb) c
+                        /mntnfs/med_data5/chenghao/fresh_eval/src/fun_base_fill_LLM.py:324: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
+                                '''
+                            else:
+                                outputs = model(batch_input_ids)
+
+                                # 取出模型的logits
+                                if 'chatglm3-6b' in model_name:   
+                                    logits = outputs.logits.float()
+                                else:
+                                    logits = outputs.logits
+
+                                loss = torch.nn.functional.cross_entropy(logits[:, :-1, :].view(-1, logits.shape[-1]), batch_input_ids[:,1:].view(-1), reduction='none')
+                            
+
+                        loss_sum = loss.sum()
+                        loss_mean = loss.mean()
+                        losses_list = loss.tolist()
+
+                        # 准备要写入日志的数据
+                        tmp_dic = {
+                            'model_name': model_name,
+                            'file_name': file_name,
+                            'lengths': len(batch_input_ids[0]),
+                            'length_str':len(corpus[enu][:max_str_len]),
+                            'loss_sum': loss_sum.item(),  # 转换为Python标准数据类型
+                            'loss_mean': loss_mean.item(),
+                            'losses_list': losses_list
+                        }
+                        import json
+                        with open(config['detail_log_path'], 'a') as f:
+                        
+                            json.dump(tmp_dic, f)
+                            f.write("\n")
+
+                        total_loss += loss.sum().item()
+                        total_tokens += batch_input_ids.numel()
+
+                # 计算每个类别的平均损失
+                # pdb.set_trace()
+                average_loss = total_loss / total_tokens
+                avg_str_loss = total_loss/len(tokenized_corpus)
+                
+
+                print(f"{file_name} total loss:", average_loss)
+                import json
+
+                logs = {
+                "model_name": model_name,
+                "file_name": file_name,
+                "processed_sequences": len(processed_sequences), 
+                "average_loss": average_loss,
+                "avg_str_loss": avg_str_loss
+                }
+
+                # with open(f'/mntnfs/med_data5/chenghao/fresh_eval/log/year_arxiv/j_y_ans_{file_in_data_folder}.json', 'a') as f:
+                with open(config['extract_log_path'], 'a') as f:
+
+                    json.dump(logs, f)
+                    f.write("\n")
+
+                logging.info(logs)
+                
+        except Exception as e:
+            logging.error(f"{model_name}, error:{e} ,detail:{traceback.format_exc()}")
+            with open(config['extract_log_path'], 'a') as f:
+                # json.dump(logs, f)
+                f.write(f"{model_name} failed \n")
+                print(f"{model_name} failed for {e} detail:{traceback.format_exc()}\n")
+
+if __name__=='__main__':
+    config={}
+    print(file_in_data_folder)
+    file_dic_list_strings=get_text.get_text_from(file_in_data_folder,limit=limit_lines_per_file)
+    config['max_sequence_length'],config['max_str_len'],config['limit_lines_per_file']=2048,5000,10
+    config['extract_log_path']=f'/mntnfs/med_data5/chenghao/fresh_eval/log/test_fun_dev/extract.log'
+    config['detail_log_path']=f'/mntnfs/med_data5/chenghao/fresh_eval/log/test_fun_dev/detail.log'
+
+    config['model_path']='/mntnfs/med_data5/liangjuhao/models/TinyLlama-1.1B-Chat-v0.6'#paths[:1]
+    config['batch']=16
+    config['model_type']='HF'
+
+    print('start',config['model_path'])
+    config['file_dic_list_strings']=file_dic_list_strings
+    run_model_on_dic(config)

gradio_cached_examples/186/log.csv

@@ -0,0 +1,3 @@
+colored text,flag,username,timestamp
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:40.149274
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:42.364248

gradio_cached_examples/212/log.csv

@@ -0,0 +1,3 @@
+colored text,flag,username,timestamp
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:44.632048
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:46.813954

hello_test.py

@@ -0,0 +1,13 @@
+import gradio as gr
+
+def greet(name, intensity):
+    return "Hello, " + name + ",,!" * int(intensity)
+    # return "Hello, " + name + ",,!" * int(0/int(intensity))# you can see the bug in command line
+
+demo = gr.Interface(
+    fn=greet,
+    inputs=["text", "slider"],
+    outputs=["text"],
+)
+
+demo.launch(debug=True)

hf_space_test.py

@@ -0,0 +1,20 @@
+# this need hugginface connection
+import gradio as gr
+from transformers  import pipeline
+
+
+pipeline = pipeline(task="image-classification", model="julien-c/hotdog-not-hotdog")
+
+def predict(input_img):
+    predictions = pipeline(input_img)
+    return input_img, {p["label"]: p["score"] for p in predictions} 
+
+gradio_app = gr.Interface(
+    predict,
+    inputs=gr.Image(label="Select hot dog candidate", sources=['upload', 'webcam'], type="pil"),
+    outputs=[gr.Image(label="Processed Image"), gr.Label(label="Result", num_top_classes=2)],
+    title="Hot Dog? Or Not?",
+)
+
+if __name__ == "__main__":
+    gradio_app.launch(debug=True)

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+torch