update wording

app.py

@@ -150,7 +150,7 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 shown_columns = gr.CheckboxGroup(
                     choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden],

crm-results/hf_leaderboard_accuracy.csv

@@ -1,22 +1,22 @@
 Use Case Name,Use Case Type,Accuracy Method,Model Name,Model Version,LLM Provider,Factuality,Instruction Following,Conciseness,Completeness,Accuracy
-Service: Conversation summary,Summary,Auto,Gemini Pro 1,Gemini Pro 1,Google,3.871720116618076,3.9834791059280854,3.847424684159378,3.9193391642371234,3.9054907677356656
-Service: Conversation summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.9669582118561713,3.9961127308066082,3.9300291545189503,3.9844509232264333,3.9693877551020407
-Service: Conversation summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.9310009718172982,3.998056365403304,3.8104956268221573,3.9737609329446064,3.9283284742468414
-Service: Conversation summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.9504373177842567,4.0,3.9067055393586005,3.9805636540330416,3.9594266277939747
-Service: Conversation summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.9591836734693877,3.998056365403304,3.881438289601555,3.992225461613217,3.957725947521866
-Service: Conversation summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.9591836734693877,3.999028182701652,3.6997084548104957,3.993197278911565,3.912779397473275
-Service: Conversation summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.954324586977648,4.0,3.8328474246841595,3.9951409135082603,3.945578231292517
-Service: Conversation summary,Summary,Auto,XGen 2,XGen 2 (1228),Salesforce,3.880466472303207,3.9941690962099123,3.9047619047619047,3.836734693877551,3.9040330417881437
-Service: Conversation summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.8746355685131197,3.991253644314869,3.7862001943634596,3.9407191448007777,3.898202137998057
-Service: Conversation summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.9591836734693877,3.999028182701652,3.9280855199222544,3.990281827016521,3.9691448007774537
-Service: Conversation summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.7560738581146746,3.938775510204082,3.6530612244897958,3.938775510204082,3.8216715257531586
-Service: Conversation summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.9310009718172982,3.9961127308066082,3.9834791059280854,3.935860058309038,3.961613216715257
-Service: Conversation summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.9640427599611274,3.998056365403304,3.7657920310981536,3.989310009718173,3.9293002915451893
-Service: Conversation summary,Summary,Auto,Gemini Pro 1.5,Gemini Pro 1.5,Google,3.9494655004859087,3.9961127308066082,3.943634596695821,3.9727891156462585,3.9655004859086493
-Service: Conversation summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.922254616132167,4.0,3.565597667638484,3.9961127308066082,3.8709912536443145
-Service: Conversation summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.7764820213799806,3.927113702623907,3.327502429543246,3.924198250728863,3.738824101068999
-Service: Conversation summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.9115646258503403,3.987366375121477,3.3751214771622933,3.9825072886297375,3.814139941690962
-Service: Conversation summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.9037900874635567,3.997084548104956,3.7755102040816326,3.9280855199222544,3.9011175898931
+Service: Conversation Summary,Summary,Auto,Gemini Pro 1,Gemini Pro 1,Google,3.871720116618076,3.9834791059280854,3.847424684159378,3.9193391642371234,3.9054907677356656
+Service: Conversation Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.9669582118561713,3.9961127308066082,3.9300291545189503,3.9844509232264333,3.9693877551020407
+Service: Conversation Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.9310009718172982,3.998056365403304,3.8104956268221573,3.9737609329446064,3.9283284742468414
+Service: Conversation Summary,Summary,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.9504373177842567,4.0,3.9067055393586005,3.9805636540330416,3.9594266277939747
+Service: Conversation Summary,Summary,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.9591836734693877,3.998056365403304,3.881438289601555,3.992225461613217,3.957725947521866
+Service: Conversation Summary,Summary,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.9591836734693877,3.999028182701652,3.6997084548104957,3.993197278911565,3.912779397473275
+Service: Conversation Summary,Summary,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.954324586977648,4.0,3.8328474246841595,3.9951409135082603,3.945578231292517
+Service: Conversation Summary,Summary,Auto,XGen 2,XGen 2 (1228),Salesforce,3.880466472303207,3.9941690962099123,3.9047619047619047,3.836734693877551,3.9040330417881437
+Service: Conversation Summary,Summary,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.8746355685131197,3.991253644314869,3.7862001943634596,3.9407191448007777,3.898202137998057
+Service: Conversation Summary,Summary,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.9591836734693877,3.999028182701652,3.9280855199222544,3.990281827016521,3.9691448007774537
+Service: Conversation Summary,Summary,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.7560738581146746,3.938775510204082,3.6530612244897958,3.938775510204082,3.8216715257531586
+Service: Conversation Summary,Summary,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.9310009718172982,3.9961127308066082,3.9834791059280854,3.935860058309038,3.961613216715257
+Service: Conversation Summary,Summary,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.9640427599611274,3.998056365403304,3.7657920310981536,3.989310009718173,3.9293002915451893
+Service: Conversation Summary,Summary,Auto,Gemini Pro 1.5,Gemini Pro 1.5,Google,3.9494655004859087,3.9961127308066082,3.943634596695821,3.9727891156462585,3.9655004859086493
+Service: Conversation Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.922254616132167,4.0,3.565597667638484,3.9961127308066082,3.8709912536443145
+Service: Conversation Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.7764820213799806,3.927113702623907,3.327502429543246,3.924198250728863,3.738824101068999
+Service: Conversation Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.9115646258503403,3.987366375121477,3.3751214771622933,3.9825072886297375,3.814139941690962
+Service: Conversation Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.9037900874635567,3.997084548104956,3.7755102040816326,3.9280855199222544,3.9011175898931
 Sales: Email Generation,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,2.2452830188679247,1.9811320754716981,1.8867924528301887,2.018867924528302,2.0330188679245285
 Sales: Email Generation,Generation,Auto,Gemini Pro 1,Gemini Pro 1,Google,3.188679245283019,3.207547169811321,3.0754716981132075,3.0377358490566038,3.1273584905660377
 Sales: Email Generation,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.660377358490566,3.7358490566037736,3.5849056603773586,3.0754716981132075,3.5141509433962264
@@ -107,24 +107,24 @@ Service: Email Summary,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.6
 Service: Email Summary,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.642857142857143,3.8877551020408165,3.520408163265306,3.8979591836734695,3.737244897959184
 Service: Email Summary,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.673469387755102,3.979591836734694,3.7346938775510203,3.9591836734693877,3.836734693877551
 Service: Email Summary,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.377551020408163,3.377551020408163,3.36734693877551,3.6530612244897958,3.443877551020408
-Service: Knowledge creation from Case Info,Generation,Auto,Gemini Pro 1,Gemini Pro 1,Google,3.0,3.1875,2.8125,3.0625,3.015625
-Service: Knowledge creation from Case Info,Generation,Auto,GPT4-o,GPT4-o,OpenAI,3.3125,3.625,3.1875,3.6875,3.453125
-Service: Knowledge creation from Case Info,Generation,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.125,3.125,2.8125,3.125,3.046875
-Service: Knowledge creation from Case Info,Generation,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.1875,3.5625,3.25,3.4375,3.359375
-Service: Knowledge creation from Case Info,Generation,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.125,3.5625,3.1875,3.25,3.28125
-Service: Knowledge creation from Case Info,Generation,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.0,3.5625,3.125,3.1875,3.21875
-Service: Knowledge creation from Case Info,Generation,Auto,XGen 2,XGen 2 (1228),Salesforce,3.1875,3.25,3.0,3.1875,3.15625
-Service: Knowledge creation from Case Info,Generation,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.3125,3.5625,3.0625,3.375,3.328125
-Service: Knowledge creation from Case Info,Generation,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.0625,2.4375,2.0625,2.3125,2.46875
-Service: Knowledge creation from Case Info,Generation,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.125,3.0625,2.8125,3.0625,3.015625
-Service: Knowledge creation from Case Info,Generation,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.3125,3.6875,3.1875,3.375,3.390625
-Service: Knowledge creation from Case Info,Generation,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.25,3.5625,3.3125,3.4375,3.390625
-Service: Knowledge creation from Case Info,Generation,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.125,3.375,2.9375,3.1875,3.15625
-Service: Knowledge creation from Case Info,Generation,Auto,Gemini Pro 1.5,Gemini Pro 1.5,Google,3.1875,3.4375,3.125,3.125,3.21875
-Service: Knowledge creation from Case Info,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.1875,3.5625,3.0,3.4375,3.296875
-Service: Knowledge creation from Case Info,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.0625,3.5625,2.9375,3.375,3.234375
-Service: Knowledge creation from Case Info,Generation,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.25,3.4375,3.125,3.375,3.296875
-Service: Knowledge creation from Case Info,Generation,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.125,3.4375,3.0,3.3125,3.21875
+Service: Knowledge Creation from Case Info,Generation,Auto,Gemini Pro 1,Gemini Pro 1,Google,3.0,3.1875,2.8125,3.0625,3.015625
+Service: Knowledge Creation from Case Info,Generation,Auto,GPT4-o,GPT4-o,OpenAI,3.3125,3.625,3.1875,3.6875,3.453125
+Service: Knowledge Creation from Case Info,Generation,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.125,3.125,2.8125,3.125,3.046875
+Service: Knowledge Creation from Case Info,Generation,Auto,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.1875,3.5625,3.25,3.4375,3.359375
+Service: Knowledge Creation from Case Info,Generation,Auto,Claude 3 Haiku,Claude 3 Haiku,Anthropic,3.125,3.5625,3.1875,3.25,3.28125
+Service: Knowledge Creation from Case Info,Generation,Auto,Cohere Command R+,cohere.cmd-R+,Cohere AI,3.0,3.5625,3.125,3.1875,3.21875
+Service: Knowledge Creation from Case Info,Generation,Auto,XGen 2,XGen 2 (1228),Salesforce,3.1875,3.25,3.0,3.1875,3.15625
+Service: Knowledge Creation from Case Info,Generation,Auto,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.3125,3.5625,3.0625,3.375,3.328125
+Service: Knowledge Creation from Case Info,Generation,Auto,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.0625,2.4375,2.0625,2.3125,2.46875
+Service: Knowledge Creation from Case Info,Generation,Auto,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.125,3.0625,2.8125,3.0625,3.015625
+Service: Knowledge Creation from Case Info,Generation,Auto,GPT 4 Turbo,gpt-4-0613,OpenAI,3.3125,3.6875,3.1875,3.375,3.390625
+Service: Knowledge Creation from Case Info,Generation,Auto,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.25,3.5625,3.3125,3.4375,3.390625
+Service: Knowledge Creation from Case Info,Generation,Auto,Claude 3 Opus,Claude 3 (Opus),Anthropic,3.125,3.375,2.9375,3.1875,3.15625
+Service: Knowledge Creation from Case Info,Generation,Auto,Gemini Pro 1.5,Gemini Pro 1.5,Google,3.1875,3.4375,3.125,3.125,3.21875
+Service: Knowledge Creation from Case Info,Generation,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.1875,3.5625,3.0,3.4375,3.296875
+Service: Knowledge Creation from Case Info,Generation,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.0625,3.5625,2.9375,3.375,3.234375
+Service: Knowledge Creation from Case Info,Generation,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.25,3.4375,3.125,3.375,3.296875
+Service: Knowledge Creation from Case Info,Generation,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.125,3.4375,3.0,3.3125,3.21875
 Sales: Email Summary,Summary,Auto,Gemini Pro 1,Gemini Pro 1,Google,3.377551020408163,3.3877551020408165,3.193877551020408,3.4285714285714284,3.346938775510204
 Sales: Email Summary,Summary,Auto,GPT4-o,GPT4-o,OpenAI,3.8877551020408165,3.9693877551020407,3.86734693877551,3.8979591836734695,3.9056122448979593
 Sales: Email Summary,Summary,Auto,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.5714285714285716,3.8979591836734695,3.9183673469387754,3.663265306122449,3.7627551020408165
@@ -197,21 +197,21 @@ Service: Live Chat Insights,Summary,Auto,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Met
 Service: Live Chat Insights,Summary,Auto,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.8125,3.9921875,3.7734375,3.875,3.86328125
 Service: Live Chat Insights,Summary,Auto,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.8046875,3.96875,3.6015625,3.953125,3.83203125
 Service: Live Chat Insights,Summary,Auto,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.80859375,3.9765625,3.92578125,3.7734375,3.87109375
-Service: Knowledge creation from Case Info,Generation,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.3125,3.1250000000000004,3.15625,3.0208333333333335,3.1536458333333335
-Service: Knowledge creation from Case Info,Generation,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.2604166666666665,3.078125,3.3125,3.375,3.2565104166666665
-Service: Knowledge creation from Case Info,Generation,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.354166666666667,3.1197916666666665,3.0729166666666665,3.223958333333333,3.192708333333333
-Service: Knowledge creation from Case Info,Generation,Manual,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.1770833333333335,2.9010416666666665,3.0416666666666665,2.901041666666667,3.005208333333333
-Service: Knowledge creation from Case Info,Generation,Manual,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.234375,3.0364583333333335,3.1666666666666665,3.0885416666666665,3.1315104166666665
-Service: Knowledge creation from Case Info,Generation,Manual,XGen 2,XGen 2 (1228),Salesforce,2.880208333333333,2.921875,3.0416666666666665,2.84375,2.921875
-Service: Knowledge creation from Case Info,Generation,Manual,Claude 3 Haiku,Claude 3 Haiku,Anthropic,2.979166666666667,3.1562499999999996,2.8125000000000004,2.8385416666666665,2.946614583333333
-Service: Knowledge creation from Case Info,Generation,Manual,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.03125,2.8177083333333335,3.09375,2.6822916666666665,2.90625
-Service: Knowledge creation from Case Info,Generation,Manual,Gemini Pro 1,Gemini Pro 1,Google,3.213541666666667,2.869791666666667,3.1093750000000004,2.9427083333333335,3.033854166666667
-Service: Knowledge creation from Case Info,Generation,Manual,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.0729166666666665,2.4687500000000004,2.9947916666666665,2.416666666666667,2.73828125
-Service: Knowledge creation from Case Info,Generation,Manual,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.0572916666666665,2.770833333333333,3.0520833333333335,2.8489583333333335,2.932291666666667
-Service: Knowledge creation from Case Info,Generation,Manual,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.1562500000000004,2.927083333333333,3.1197916666666665,3.0989583333333335,3.0755208333333335
-Service: Knowledge creation from Case Info,Generation,Manual,GPT4-o,GPT4-o,OpenAI,3.3020833333333335,3.296875,3.171875,3.380208333333334,3.287760416666667
-Service: Knowledge creation from Case Info,Generation,Manual,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.1510416666666665,2.9114583333333335,3.046875,2.96875,3.01953125
-Service: Knowledge creation from Case Info,Generation,Manual,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.208333333333333,3.057291666666667,3.114583333333333,3.0000000000000004,3.095052083333333
+Service: Knowledge Creation from Case Info,Generation,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.3125,3.1250000000000004,3.15625,3.0208333333333335,3.1536458333333335
+Service: Knowledge Creation from Case Info,Generation,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.2604166666666665,3.078125,3.3125,3.375,3.2565104166666665
+Service: Knowledge Creation from Case Info,Generation,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.354166666666667,3.1197916666666665,3.0729166666666665,3.223958333333333,3.192708333333333
+Service: Knowledge Creation from Case Info,Generation,Manual,AI21 Jamba-Instruct,AI21 (jamba-instruct-preview),AI21,3.1770833333333335,2.9010416666666665,3.0416666666666665,2.901041666666667,3.005208333333333
+Service: Knowledge Creation from Case Info,Generation,Manual,Mistral 7B,Mistral-7B-Instruct-v0.1,Mistral,3.234375,3.0364583333333335,3.1666666666666665,3.0885416666666665,3.1315104166666665
+Service: Knowledge Creation from Case Info,Generation,Manual,XGen 2,XGen 2 (1228),Salesforce,2.880208333333333,2.921875,3.0416666666666665,2.84375,2.921875
+Service: Knowledge Creation from Case Info,Generation,Manual,Claude 3 Haiku,Claude 3 Haiku,Anthropic,2.979166666666667,3.1562499999999996,2.8125000000000004,2.8385416666666665,2.946614583333333
+Service: Knowledge Creation from Case Info,Generation,Manual,Cohere Command Text,cohere.command-text-v14,Cohere AI,3.03125,2.8177083333333335,3.09375,2.6822916666666665,2.90625
+Service: Knowledge Creation from Case Info,Generation,Manual,Gemini Pro 1,Gemini Pro 1,Google,3.213541666666667,2.869791666666667,3.1093750000000004,2.9427083333333335,3.033854166666667
+Service: Knowledge Creation from Case Info,Generation,Manual,LLaMA 3 70B,Meta-Llama-3-70B-Instruct,Meta,3.0729166666666665,2.4687500000000004,2.9947916666666665,2.416666666666667,2.73828125
+Service: Knowledge Creation from Case Info,Generation,Manual,LLaMA 3 8B,Meta-Llama-3-8B-Instruct,Meta,3.0572916666666665,2.770833333333333,3.0520833333333335,2.8489583333333335,2.932291666666667
+Service: Knowledge Creation from Case Info,Generation,Manual,SF-TextBase 7B,TextBase-7B (Mistral FT),Salesforce,3.1562500000000004,2.927083333333333,3.1197916666666665,3.0989583333333335,3.0755208333333335
+Service: Knowledge Creation from Case Info,Generation,Manual,GPT4-o,GPT4-o,OpenAI,3.3020833333333335,3.296875,3.171875,3.380208333333334,3.287760416666667
+Service: Knowledge Creation from Case Info,Generation,Manual,SF-TextSum,Summarization model 7B for Service (Mistral FT),Salesforce,3.1510416666666665,2.9114583333333335,3.046875,2.96875,3.01953125
+Service: Knowledge Creation from Case Info,Generation,Manual,SF-TextBase 70B,TextBase-70B (Llama FT),Salesforce,3.208333333333333,3.057291666666667,3.114583333333333,3.0000000000000004,3.095052083333333
 Service: Reply Recommendations,Generation,Manual,GPT 3.5 Turbo,gpt-3.5-turbo,OpenAI,3.0935185185185183,3.22037037037037,3.443518518518519,3.0453703703703705,3.2006944444444443
 Service: Reply Recommendations,Generation,Manual,GPT 4 Turbo,gpt-4-0613,OpenAI,3.525925925925926,3.3203703703703704,3.5129629629629635,3.52962962962963,3.4722222222222228
 Service: Reply Recommendations,Generation,Manual,Mixtral 8x7B,Mixtral-8x7B-v0.1,Mistral,3.015740740740741,3.0203703703703706,3.2111111111111112,2.837037037037037,3.021064814814815

crm-results/hf_leaderboard_crm_bias.csv

@@ -1,4 +1,4 @@
-Model Name,CRM Bias
+Model Name,CRM Fairness
 LLaMA 3 70B,"98.3% [98.2%, 98.5%]"
 SF-TextBase 70B,"98.2% [98.0%, 98.4%]"
 Claude 3 Opus,"97.8% [97.4%, 98.1%]"
@@ -16,4 +16,4 @@ Cohere Command Text,"95.2% [95.0%, 95.3%]"
 LLaMA 3 8B,"95.1% [94.8%, 95.5%]"
 Mixtral 8x7B,"94.9% [94.6%, 95.1%]"
 SF-TextBase 7B,"94.6% [94.1%, 95.1%]"
-SF-TextSum,"93.9% [93.3%, 94.4%]"
+SF-TextSum,"93.9% [93.3%, 94.4%]"

crm-results/hf_leaderboard_flavor_mapping.csv

@@ -1,5 +1,5 @@
 Use Case Name,Use Case Type,Cost and Speed: Flavor
-Service: Conversation summary,Summary,Short
+Service: Conversation Summary,Summary,Short
 Service: Reply Recommendations,Generation,Short
 Sales: Email Generation,Generation,Short
 Sales & Service: Update CRM Info,Generation,Long
@@ -8,5 +8,5 @@ Sales: Call Summary,Summary,Long
 Service: Live Chat Insights,Summary,Short
 Service: Live Chat Summary,Summary,Long
 Service: Email Summary,Summary,Long
-Service: Knowledge creation from Case Info,Generation,Long
-Sales: Email Summary,Summary,Long
+Service: Knowledge Creation from Case Info,Generation,Long
+Sales: Email Summary,Summary,Long

src/about.py

@@ -53,7 +53,7 @@ LLM_BENCHMARKS_TEXT = """
 9) Building a reliable LLM-based evaluator remains an open challenge due to inherent biases such as 1) Length Bias: the tendency to favor longer responses, and 2) Self-enhancement Bias: the tendency of the LLM-evaluator to favor its own responses.
 10) Task-specific model variants were not used from the external providers (command-r is sort of retrieval specific, but this was not one of the use cases).
 11) Maybe something about the tasks being primarily summarization / generation
-12) Trust & Safety was benchmarked on public datasets as well as bias perturbations on CRM datasets. For gender bias, person names and pronouns were perturbed. For company bias, company names were perturbed to competitors in the same sector. For the CRM Bias metric, higher means less bias.
+12) Trust & Safety was benchmarked on public datasets as well as bias perturbations on CRM datasets. For gender bias, person names and pronouns were perturbed. For company bias, company names were perturbed to competitors in the same sector. For the CRM Fairness metric, higher means less bias.
 13) Cost per request for self-hosted models assume a minimal frequency of calling the model, since the costs are per hour. All latencies / cost assume a single user at a time.
 14) The current auto-evaluation is based on LLaMA-70B as Judge, which showed the highest correlation with human annotaotors.
 """

src/display/utils.py

@@ -28,7 +28,9 @@ auto_eval_column_dict.append(
 auto_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
 auto_eval_column_dict.append(["use_case_name", ColumnContent, ColumnContent("Use Case Name", "markdown", True)])
 auto_eval_column_dict.append(["use_case_type", ColumnContent, ColumnContent("Use Case Type", "markdown", False)])
-auto_eval_column_dict.append(["accuracy_method", ColumnContent, ColumnContent("Accuracy Method", "markdown", False)])
+auto_eval_column_dict.append(
+    ["accuracy_method", ColumnContent, ColumnContent("Accuracy Method", "markdown", False, never_hidden=True)]
+)
 # Accuracy metrics
 auto_eval_column_dict.append(["accuracy_metric_average", ColumnContent, ColumnContent("Accuracy", "markdown", True)])
 auto_eval_column_dict.append(
@@ -58,7 +60,7 @@ auto_eval_column_dict.append(["ts", ColumnContent, ColumnContent("Trust & Safety
 auto_eval_column_dict.append(["safety", ColumnContent, ColumnContent("Safety", "markdown", False)])
 auto_eval_column_dict.append(["privacy", ColumnContent, ColumnContent("Privacy", "markdown", False)])
 auto_eval_column_dict.append(["truthfulness", ColumnContent, ColumnContent("Truthfulness", "markdown", False)])
-auto_eval_column_dict.append(["crm_bias", ColumnContent, ColumnContent("CRM Bias", "markdown", False)])
+auto_eval_column_dict.append(["crm_fairness", ColumnContent, ColumnContent("CRM Fairness", "markdown", False)])
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
@@ -89,7 +91,7 @@ ts_eval_column_dict.append(["ts", ColumnContent, ColumnContent("Trust & Safety",
 ts_eval_column_dict.append(["safety", ColumnContent, ColumnContent("Safety", "markdown", False)])
 ts_eval_column_dict.append(["privacy", ColumnContent, ColumnContent("Privacy", "markdown", False)])
 ts_eval_column_dict.append(["truthfulness", ColumnContent, ColumnContent("Truthfulness", "markdown", False)])
-ts_eval_column_dict.append(["crm_bias", ColumnContent, ColumnContent("CRM Bias", "markdown", False)])
+ts_eval_column_dict.append(["crm_fairness", ColumnContent, ColumnContent("CRM Fairness", "markdown", False)])
 # ts_eval_column_dict.append(["bias_no_ci", ColumnContent, ColumnContent("Bias No CI", "markdown", True)])
 TSEvalColumn = make_dataclass("TSEvalColumn", ts_eval_column_dict, frozen=True)
 

src/populate.py

@@ -42,7 +42,7 @@ def get_leaderboard_df_crm(
     ].apply(lambda x: x.str.rstrip("%").astype("float") / 100.0, axis=1)
 
     leaderboard_ts_df["Privacy"] = privacy_cols.mean(axis=1).transform(lambda x: "{:,.2%}".format(x))
-    leaderboard_ts_df["Bias No CI"] = leaderboard_ts_df["CRM Bias"].transform(lambda x: x.split(" ")[0])
+    leaderboard_ts_df["Bias No CI"] = leaderboard_ts_df["CRM Fairness"].transform(lambda x: x.split(" ")[0])
 
     ts_lvl2_cols = leaderboard_ts_df[
         [