Upload 2 files

generation_test_hf_script.py

@@ -0,0 +1,90 @@
+
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+
+def load_rag_benchmark_tester_ds():
+
+    # pull 200 question rag benchmark test dataset from LLMWare HuggingFace repo
+    from datasets import load_dataset
+
+    ds_name = "llmware/rag_instruct_benchmark_tester"
+
+    dataset = load_dataset(ds_name)
+
+    print("update: loading RAG Benchmark test dataset - ", dataset)
+
+    test_set = []
+    for i, samples in enumerate(dataset["train"]):
+        test_set.append(samples)
+
+        # to view test set samples
+        # print("rag benchmark dataset test samples: ", i, samples)
+
+    return test_set
+
+
+def run_test(model_name, test_ds):
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    print("\nRAG Performance Test - 200 questions")  
+    print("update: model - ", model_name)
+    print("update: device - ", device)
+
+    model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+    model.to(device)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+    for i, entries in enumerate(test_ds):
+
+        # prepare prompt packaging used in fine-tuning process
+        # note: in our testing, Yi model performed better with trailing "\n" at end of prompt
+        new_prompt = "<human>: " + entries["context"] + "\n" + entries["query"] + "\n" + "<bot>:" + "\n"
+
+        inputs = tokenizer(new_prompt, return_tensors="pt")
+        start_of_output = len(inputs.input_ids[0])
+
+        #   temperature: set at 0.3 for consistency of output
+        #   max_new_tokens:  set at 100 - may prematurely stop a few of the summaries
+
+        outputs = model.generate(
+            inputs.input_ids.to(device),
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.eos_token_id,
+            do_sample=True,
+            temperature=0.3,
+            max_new_tokens=100,
+            )
+
+        output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True)
+
+        # quick/optional post-processing clean-up of potential fine-tuning artifacts
+
+        eot = output_only.find("<|endoftext|>")
+        if eot > -1:
+            output_only = output_only[:eot]
+
+        bot = output_only.find("<bot>:")
+        if bot > -1:
+            output_only = output_only[bot+len("<bot>:"):]
+
+        # end - post-processing
+
+        print("\n")
+        print(i, "llm_response - ", output_only)
+        print(i, "gold_answer - ", entries["answer"])
+
+    return 0
+
+
+if __name__ == "__main__":
+
+    test_ds = load_rag_benchmark_tester_ds()
+
+    model_name = "llmware/dragon-deci-7b-v0"
+    output = run_test(model_name,test_ds)
+
+

generation_test_llmware_script.py

@@ -0,0 +1,70 @@
+
+from llmware.prompts import Prompt
+
+
+def load_rag_benchmark_tester_ds():
+
+    # pull 200 question rag benchmark test dataset from LLMWare HuggingFace repo
+    from datasets import load_dataset
+
+    ds_name = "llmware/rag_instruct_benchmark_tester"
+
+    dataset = load_dataset(ds_name)
+
+    print("update: loading RAG Benchmark test dataset - ", dataset)
+
+    test_set = []
+    for i, samples in enumerate(dataset["train"]):
+        test_set.append(samples)
+
+        # to view test set samples
+        # print("rag benchmark dataset test samples: ", i, samples)
+
+    return test_set
+
+
+def run_test(model_name, prompt_list):
+
+    print("\nupdate: Starting RAG Benchmark Inference Test - ", model_name)
+
+    # pull DRAGON / BLING model directly from catalog, e.g., no from_hf=True
+    prompter = Prompt().load_model(model_name)
+
+    for i, entries in enumerate(prompt_list):
+
+        prompt = entries["query"]
+        context = entries["context"]
+
+        response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3)
+
+        print("\nupdate: model inference output - ", i, response["llm_response"])
+        print("update: gold_answer              - ", i, entries["answer"])
+
+        fc = prompter.evidence_check_numbers(response)
+        sc = prompter.evidence_comparison_stats(response)
+        sr = prompter.evidence_check_sources(response)
+
+        print("\nFact-Checking Tools")
+
+        for entries in fc:
+            for f, facts in enumerate(entries["fact_check"]):
+                print("update: fact check - ", f, facts)
+
+        for entries in sc:
+            print("update: comparison stats - ", entries["comparison_stats"])
+
+        for entries in sr:
+            for s, sources in enumerate(entries["source_review"]):
+                print("update: sources - ", s, sources)
+
+    return 0
+
+
+if __name__ == "__main__":
+
+    core_test_set = load_rag_benchmark_tester_ds()
+
+    # one of the 7 gpu dragon models
+    gpu_model_name = "llmware/dragon-deci-7b-v0"
+
+    output = run_test(gpu_model_name, core_test_set)