Repository: truera/trulens
Issue: Getting empty results from leaderboard and records even though I can see the feedback results in the print statements
Root cause: Documentation Deficiency

`
import pandas as pd
import httpx
from datetime import datetime
import os


from trulens.apps.virtual import VirtualApp
from trulens.core import Feedback
from trulens.providers.openai.provider import AzureOpenAI as AzureOpenAITrulens
from trulens.apps.virtual import TruVirtual
from trulens.core import TruSession
from trulens.dashboard import run_dashboard
from trulens.core.database.connector.default import DefaultDBConnector
from trulens.feedback.groundtruth import GroundTruthAgreement


import skai_rag_query
import skai_rag_query.asynchronous.rag_query
import skai_rag_query.settings
from skai_rag_query.rag_query_config import get_settings

from initialize_logging import init_logger

analytics_results_directory = os.getenv("ANALYTICS_RESULTS_DIRECTORY")
ts = datetime.now().strftime("%Y-%m-%d_%H_%M_%S")
fname = ts + "_skai_eval.log"
log_filename = os.path.join(analytics_results_directory, fname)
logger_name = "root"
logger = init_logger(logger_name, log_filename)

sqlite_file_absolute_path = os.path.join(
    analytics_results_directory, os.getenv("TRULENS_SQLITE_FILE_NAME")
)
logger.info(f"Before instantiating Tru {sqlite_file_absolute_path=}")

app_run_name = f"{os.getenv('PAGE_NODE_COLLECTION')}_{ts}"
logger.info(f"app_run_name: {app_run_name}")

def main():

    data = {
        "query": ["Where is Germany?", "What is the capital of France?"],
        "response": ["Germany is in Europe", "The capital of France is Paris"],
        "contexts": [
            ["Germany is a country located in Europe."],
            [
                "France is a country in Europe and its capital is Paris.",
                "Germany is a country located in Europe",
            ],
        ],
    }

    golden_set = [
        {"query": "Where is Germany?", "expected_response": "Germany is in Europe"},
        {
            "query": "What is the capital of France?",
            "expected_response": "The capital of France is Paris",
        },
    ]

    df = pd.DataFrame(data)
    print(df.head())

    tru_session = TruSession()
    tru_session.reset_database()

    # Initialize provider class
    openai_args = {} # private end point details

    trulens_openai_provider = AzureOpenAITrulens(
        deployment_name=os.getenv("LLM_DEPLOYMENT_ID_PRIMARY"),
        **openai_args,
    )

    # Select context to be used in feedback.
    context = VirtualApp.select_context()

    # Question/statement relevance between question and each context chunk.
    f_context_relevance = (
        Feedback(
            trulens_openai_provider.context_relevance_with_cot_reasons,
            name="Context-Relevance",
        )
        .on_input()
        .on(context)
    )

    # Define a groundedness feedback function
    f_groundedness = (
        Feedback(
            trulens_openai_provider.groundedness_measure_with_cot_reasons,
            name="Groundedness",
        )
        .on(context.collect())  # collect context chunks into a list
        .on_output()
    )

    # Question/answer relevance between overall question and answer.
    f_qa_relevance = Feedback(
        trulens_openai_provider.relevance_with_cot_reasons, name="Answer-Relevance"
    ).on_input_output()

    f_groundtruth = Feedback(
        GroundTruthAgreement(
            golden_set, provider=trulens_openai_provider
        ).agreement_measure,
        name="GroundTruth",
    ).on_input_output()

    # create an instance of the VirtualApp class from  trulens
    virtual_app = VirtualApp()
    virtual_recorder = TruVirtual(
        app_name=app_run_name,
        app_version="simple",
        app=virtual_app,
        feedbacks=[f_context_relevance, f_qa_relevance, f_groundedness, f_groundtruth],
    )

    virtual_records = virtual_recorder.add_dataframe(df)

    for rec in virtual_records:
        print(rec.main_input, "-->", rec.main_output)

        for feedback, feedback_result in rec.wait_for_feedback_results().items():
            print("\t", feedback.name, feedback_result.result)

        print()

    print(tru_session.get_records_and_feedback(app_ids=[app_run_name]))
    print(tru_session.get_leaderboard(app_ids=[app_run_name]))

    leaderboard = tru_session.get_leaderboard(app_ids=[app_run_name])
    leaderboard.to_csv(
        os.path.join(
            analytics_results_directory, f"QA_board_results_{app_run_name}.csv"
        ),
        index=False,
    )

    records, feedback = tru_session.get_records_and_feedback(app_ids=[app_run_name])
    records.to_csv(
        os.path.join(
            analytics_results_directory, f"QA_records_results_{app_run_name}.csv"
        ),
        index=False,
    )

if __name__ == "__main__":
    main()
`

## print output details:

Where is Germany? --> Germany is in Europe
         Context-Relevance 1.0
         Answer-Relevance 1.0
         Groundedness 1.0
         GroundTruth 1.0

What is the capital of France? --> The capital of France is Paris
         Context-Relevance 0.5
         Answer-Relevance 1.0
         Groundedness 1.0
         GroundTruth 1.0

## leaderboard dataframe is empty
(Empty DataFrame
Columns: [app_id, app_json, type, record_id, input, output, tags, record_json, cost_json, perf_json, ts, latency, total_tokens, total_cost, cost_currency, app_name, 
app_version]
Index: [], [])

# record dataframe is empty
Empty DataFrame
Columns: [latency, total_cost]
Index: []

Hi,

The issue here appears to be that you’re writing the leaderboard/records to
csv before the feedback is computed. Please wait until feedback is computed
before writing out.

Let me know if the issue persists after making this change.

-Josh

On Fri, Mar 21, 2025 at 3:57 AM Sudhakar ***@***.***> wrote:

> `
> import pandas as pd
> import httpx
> from datetime import datetime
> import os
>
> from trulens.apps.virtual import VirtualApp
> from trulens.core import Feedback
> from trulens.providers.openai.provider import AzureOpenAI as
> AzureOpenAITrulens
> from trulens.apps.virtual import TruVirtual
> from trulens.core import TruSession
> from trulens.dashboard import run_dashboard
> from trulens.core.database.connector.default import DefaultDBConnector
> from trulens.feedback.groundtruth import GroundTruthAgreement
>
> import skai_rag_query
> import skai_rag_query.asynchronous.rag_query
> import skai_rag_query.settings
> from skai_rag_query.rag_query_config import get_settings
>
> from initialize_logging import init_logger
>
> analytics_results_directory = os.getenv("ANALYTICS_RESULTS_DIRECTORY")
> ts = datetime.now().strftime("%Y-%m-%d_%H_%M_%S")
> fname = ts + "_skai_eval.log"
> log_filename = os.path.join(analytics_results_directory, fname)
> logger_name = "root"
> logger = init_logger(logger_name, log_filename)
>
> sqlite_file_absolute_path = os.path.join(
> analytics_results_directory, os.getenv("TRULENS_SQLITE_FILE_NAME")
> )
> logger.info(f"Before instantiating Tru {sqlite_file_absolute_path=}")
>
> app_run_name = f"{os.getenv('PAGE_NODE_COLLECTION')}_{ts}"
> logger.info(f"app_run_name: {app_run_name}")
>
> def main():
>
> data = {
>     "query": ["Where is Germany?", "What is the capital of France?"],
>     "response": ["Germany is in Europe", "The capital of France is Paris"],
>     "contexts": [
>         ["Germany is a country located in Europe."],
>         [
>             "France is a country in Europe and its capital is Paris.",
>             "Germany is a country located in Europe",
>         ],
>     ],
> }
>
> golden_set = [
>     {"query": "Where is Germany?", "expected_response": "Germany is in Europe"},
>     {
>         "query": "What is the capital of France?",
>         "expected_response": "The capital of France is Paris",
>     },
> ]
>
> df = pd.DataFrame(data)
> print(df.head())
>
> tru_session = TruSession()
> tru_session.reset_database()
>
> # Initialize provider class
> openai_args = {} # private end point details
>
> trulens_openai_provider = AzureOpenAITrulens(
>     deployment_name=os.getenv("LLM_DEPLOYMENT_ID_PRIMARY"),
>     **openai_args,
> )
>
> # Select context to be used in feedback.
> context = VirtualApp.select_context()
>
> # Question/statement relevance between question and each context chunk.
> f_context_relevance = (
>     Feedback(
>         trulens_openai_provider.context_relevance_with_cot_reasons,
>         name="Context-Relevance",
>     )
>     .on_input()
>     .on(context)
> )
>
> # Define a groundedness feedback function
> f_groundedness = (
>     Feedback(
>         trulens_openai_provider.groundedness_measure_with_cot_reasons,
>         name="Groundedness",
>     )
>     .on(context.collect())  # collect context chunks into a list
>     .on_output()
> )
>
> # Question/answer relevance between overall question and answer.
> f_qa_relevance = Feedback(
>     trulens_openai_provider.relevance_with_cot_reasons, name="Answer-Relevance"
> ).on_input_output()
>
> f_groundtruth = Feedback(
>     GroundTruthAgreement(
>         golden_set, provider=trulens_openai_provider
>     ).agreement_measure,
>     name="GroundTruth",
> ).on_input_output()
>
> # create an instance of the VirtualApp class from  trulens
> virtual_app = VirtualApp()
> virtual_recorder = TruVirtual(
>     app_name=app_run_name,
>     app_version="simple",
>     app=virtual_app,
>     feedbacks=[f_context_relevance, f_qa_relevance, f_groundedness, f_groundtruth],
> )
>
> virtual_records = virtual_recorder.add_dataframe(df)
>
> for rec in virtual_records:
>     print(rec.main_input, "-->", rec.main_output)
>
>     for feedback, feedback_result in rec.wait_for_feedback_results().items():
>         print("\t", feedback.name, feedback_result.result)
>
>     print()
>
> print(tru_session.get_records_and_feedback(app_ids=[app_run_name]))
> print(tru_session.get_leaderboard(app_ids=[app_run_name]))
>
> leaderboard = tru_session.get_leaderboard(app_ids=[app_run_name])
> leaderboard.to_csv(
>     os.path.join(
>         analytics_results_directory, f"QA_board_results_{app_run_name}.csv"
>     ),
>     index=False,
> )
>
> records, feedback = tru_session.get_records_and_feedback(app_ids=[app_run_name])
> records.to_csv(
>     os.path.join(
>         analytics_results_directory, f"QA_records_results_{app_run_name}.csv"
>     ),
>     index=False,
> )
>
> if *name* == "*main*":
> main()
> `
> print output details:
>
> Where is Germany? --> Germany is in Europe
> Context-Relevance 1.0
> Answer-Relevance 1.0
> Groundedness 1.0
> GroundTruth 1.0
>
> What is the capital of France? --> The capital of France is Paris
> Context-Relevance 0.5
> Answer-Relevance 1.0
> Groundedness 1.0
> GroundTruth 1.0
> leaderboard dataframe is empty
>
> (Empty DataFrame
> Columns: [app_id, app_json, type, record_id, input, output, tags,
> record_json, cost_json, perf_json, ts, latency, total_tokens, total_cost,
> cost_currency, app_name,
> app_version]
> Index: [], [])
> record dataframe is empty
>
> Empty DataFrame
> Columns: [latency, total_cost]
> Index: []
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/truera/trulens/issues/1884>, or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AORAKDW7TMZGZT4LP5Z5AHL2VPWCZAVCNFSM6AAAAABZPWFFYSVHI2DSMVQWIX3LMV43ASLTON2WKOZSHEZTQMBSHA4TAMA>
> .
> You are receiving this because you are subscribed to this thread.Message
> ID: ***@***.***>
> [image: Sudhakar17]*Sudhakar17* created an issue (truera/trulens#1884)
> <https://github.com/truera/trulens/issues/1884>
>
> `
> import pandas as pd
> import httpx
> from datetime import datetime
> import os
>
> from trulens.apps.virtual import VirtualApp
> from trulens.core import Feedback
> from trulens.providers.openai.provider import AzureOpenAI as
> AzureOpenAITrulens
> from trulens.apps.virtual import TruVirtual
> from trulens.core import TruSession
> from trulens.dashboard import run_dashboard
> from trulens.core.database.connector.default import DefaultDBConnector
> from trulens.feedback.groundtruth import GroundTruthAgreement
>
> import skai_rag_query
> import skai_rag_query.asynchronous.rag_query
> import skai_rag_query.settings
> from skai_rag_query.rag_query_config import get_settings
>
> from initialize_logging import init_logger
>
> analytics_results_directory = os.getenv("ANALYTICS_RESULTS_DIRECTORY")
> ts = datetime.now().strftime("%Y-%m-%d_%H_%M_%S")
> fname = ts + "_skai_eval.log"
> log_filename = os.path.join(analytics_results_directory, fname)
> logger_name = "root"
> logger = init_logger(logger_name, log_filename)
>
> sqlite_file_absolute_path = os.path.join(
> analytics_results_directory, os.getenv("TRULENS_SQLITE_FILE_NAME")
> )
> logger.info(f"Before instantiating Tru {sqlite_file_absolute_path=}")
>
> app_run_name = f"{os.getenv('PAGE_NODE_COLLECTION')}_{ts}"
> logger.info(f"app_run_name: {app_run_name}")
>
> def main():
>
> data = {
>     "query": ["Where is Germany?", "What is the capital of France?"],
>     "response": ["Germany is in Europe", "The capital of France is Paris"],
>     "contexts": [
>         ["Germany is a country located in Europe."],
>         [
>             "France is a country in Europe and its capital is Paris.",
>             "Germany is a country located in Europe",
>         ],
>     ],
> }
>
> golden_set = [
>     {"query": "Where is Germany?", "expected_response": "Germany is in Europe"},
>     {
>         "query": "What is the capital of France?",
>         "expected_response": "The capital of France is Paris",
>     },
> ]
>
> df = pd.DataFrame(data)
> print(df.head())
>
> tru_session = TruSession()
> tru_session.reset_database()
>
> # Initialize provider class
> openai_args = {} # private end point details
>
> trulens_openai_provider = AzureOpenAITrulens(
>     deployment_name=os.getenv("LLM_DEPLOYMENT_ID_PRIMARY"),
>     **openai_args,
> )
>
> # Select context to be used in feedback.
> context = VirtualApp.select_context()
>
> # Question/statement relevance between question and each context chunk.
> f_context_relevance = (
>     Feedback(
>         trulens_openai_provider.context_relevance_with_cot_reasons,
>         name="Context-Relevance",
>     )
>     .on_input()
>     .on(context)
> )
>
> # Define a groundedness feedback function
> f_groundedness = (
>     Feedback(
>         trulens_openai_provider.groundedness_measure_with_cot_reasons,
>         name="Groundedness",
>     )
>     .on(context.collect())  # collect context chunks into a list
>     .on_output()
> )
>
> # Question/answer relevance between overall question and answer.
> f_qa_relevance = Feedback(
>     trulens_openai_provider.relevance_with_cot_reasons, name="Answer-Relevance"
> ).on_input_output()
>
> f_groundtruth = Feedback(
>     GroundTruthAgreement(
>         golden_set, provider=trulens_openai_provider
>     ).agreement_measure,
>     name="GroundTruth",
> ).on_input_output()
>
> # create an instance of the VirtualApp class from  trulens
> virtual_app = VirtualApp()
> virtual_recorder = TruVirtual(
>     app_name=app_run_name,
>     app_version="simple",
>     app=virtual_app,
>     feedbacks=[f_context_relevance, f_qa_relevance, f_groundedness, f_groundtruth],
> )
>
> virtual_records = virtual_recorder.add_dataframe(df)
>
> for rec in virtual_records:
>     print(rec.main_input, "-->", rec.main_output)
>
>     for feedback, feedback_result in rec.wait_for_feedback_results().items():
>         print("\t", feedback.name, feedback_result.result)
>
>     print()
>
> print(tru_session.get_records_and_feedback(app_ids=[app_run_name]))
> print(tru_session.get_leaderboard(app_ids=[app_run_name]))
>
> leaderboard = tru_session.get_leaderboard(app_ids=[app_run_name])
> leaderboard.to_csv(
>     os.path.join(
>         analytics_results_directory, f"QA_board_results_{app_run_name}.csv"
>     ),
>     index=False,
> )
>
> records, feedback = tru_session.get_records_and_feedback(app_ids=[app_run_name])
> records.to_csv(
>     os.path.join(
>         analytics_results_directory, f"QA_records_results_{app_run_name}.csv"
>     ),
>     index=False,
> )
>
> if *name* == "*main*":
> main()
> `
> print output details:
>
> Where is Germany? --> Germany is in Europe
> Context-Relevance 1.0
> Answer-Relevance 1.0
> Groundedness 1.0
> GroundTruth 1.0
>
> What is the capital of France? --> The capital of France is Paris
> Context-Relevance 0.5
> Answer-Relevance 1.0
> Groundedness 1.0
> GroundTruth 1.0
> leaderboard dataframe is empty
>
> (Empty DataFrame
> Columns: [app_id, app_json, type, record_id, input, output, tags,
> record_json, cost_json, perf_json, ts, latency, total_tokens, total_cost,
> cost_currency, app_name,
> app_version]
> Index: [], [])
> record dataframe is empty
>
> Empty DataFrame
> Columns: [latency, total_cost]
> Index: []
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/truera/trulens/issues/1884>, or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AORAKDW7TMZGZT4LP5Z5AHL2VPWCZAVCNFSM6AAAAABZPWFFYSVHI2DSMVQWIX3LMV43ASLTON2WKOZSHEZTQMBSHA4TAMA>
> .
> You are receiving this because you are subscribed to this thread.Message
> ID: ***@***.***>
>


Hi Josh,

Thanks for the support.

Still, the issue persists. I have added the below line:

virtual_recorder.wait_for_feedback_results().

Q1: My assumption is that when rec.wait_for_feedback_results() was called, all the metrics were calculated for that particular query and response. It showed the results in print statement. Why do we need to add "virtual_recorder.wait_for_feedback_results()."?

Q2: Even if I add this one "virtual_recorder.wait_for_feedback_results()." Still it does not work. Any other way to fix it? or Am I missing something here?

```
virtual_records = virtual_recorder.add_dataframe(df)
for rec in virtual_records:
    print(rec.main_input, "-->", rec.main_output)

    for feedback, feedback_result in rec.wait_for_feedback_results().items():
        print("\t", feedback.name, feedback_result.result)

    print()

virtual_recorder.wait_for_feedback_results()
print(tru_session.get_records_and_feedback(app_ids=[app_run_name]))
print(tru_session.get_leaderboard(app_ids=[app_run_name]))

```

## Print details

Where is Germany? --> Germany is in Europe
         Context-Relevance 1.0
         Answer-Relevance 1.0
         Groundedness 1.0
         GroundTruth 1.0

What is the capital of France? --> The capital of France is Paris
         Context-Relevance 0.5
         Answer-Relevance 1.0
         Groundedness 1.0
         GroundTruth 1.0

(Empty DataFrame
Columns: [app_id, app_json, type, record_id, input, output, tags, record_json, cost_json, perf_json, ts, latency, total_tokens, total_cost, cost_currency, app_name, 
app_version]
Index: [], [])
Empty DataFrame
Columns: [latency, total_cost]
Index: []

Cheers,
Sudhakar

No need to specify the app id for the session. The below two lines fixed the issue. 

leaderboard = tru_session.get_leaderboard()
records, feedback = tru_session.get_records_and_feedback()