{
    "time": "241031154353",
    "results": {
        "Claude3-Sonnet": {
            "META": {
                "Method": [
                    "Claude3-Sonnet",
                    "https://aws.amazon.com/bedrock/claude/"
                ],
                "Parameters": "",
                "Org": "Anthropic",
                "OpenSource": "No",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 80.75
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 71.63
            },
            "User Behavior Alignment": {
                "Overall": 70.17
            },
            "Multi-lingual Abilities": {
                "Overall": 67.76
            }
        },
        "Claude2": {
            "META": {
                "Method": [
                    "Claude2",
                    "https://aws.amazon.com/bedrock/claude/"
                ],
                "Parameters": "",
                "Org": "Anthropic",
                "OpenSource": "No",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 75.46
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 65.5
            },
            "User Behavior Alignment": {
                "Overall": 63.53
            },
            "Multi-lingual Abilities": {
                "Overall": 65.24
            }
        },
        "ChatGPT": {
            "META": {
                "Method": [
                    "ChatGPT",
                    "https://platform.openai.com/docs/models#gpt-3-5-turbo"
                ],
                "Parameters": "",
                "Org": "OpenAI",
                "OpenSource": "No",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 75.63
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 64.97
            },
            "User Behavior Alignment": {
                "Overall": 59.79
            },
            "Multi-lingual Abilities": {
                "Overall": 60.81
            }
        },
        "LLaMA3-70B-Instruct": {
            "META": {
                "Method": [
                    "LLaMA3-70B-Instruct",
                    "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct"
                ],
                "Parameters": "70B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 75.24
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 69.29
            },
            "User Behavior Alignment": {
                "Overall": 67.67
            },
            "Multi-lingual Abilities": {
                "Overall": 62.0
            }
        },
        "QWen1.5-72B": {
            "META": {
                "Method": [
                    "QWen1.5-72B",
                    "https://huggingface.co/Qwen/Qwen1.5-72B"
                ],
                "Parameters": "72B",
                "Org": "Alibaba",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 71.67
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 68.92
            },
            "User Behavior Alignment": {
                "Overall": 64.12
            },
            "Multi-lingual Abilities": {
                "Overall": 64.84
            }
        },
        "LLaMA3-70B": {
            "META": {
                "Method": [
                    "LLaMA3-70B",
                    "https://huggingface.co/meta-llama/Meta-Llama-3-70B"
                ],
                "Parameters": "70B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 69.59
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 63.56
            },
            "User Behavior Alignment": {
                "Overall": 55.77
            },
            "Multi-lingual Abilities": {
                "Overall": 58.95
            }
        },
        "LLaMA2-70B-Chat": {
            "META": {
                "Method": [
                    "LLaMA2-70B-Chat",
                    "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf"
                ],
                "Parameters": "70B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 61.84
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 40.73
            },
            "User Behavior Alignment": {
                "Overall": 44.2
            },
            "Multi-lingual Abilities": {
                "Overall": 47.04
            }
        },
        "LLaMA2-70B": {
            "META": {
                "Method": [
                    "LLaMA2-70B",
                    "https://huggingface.co/meta-llama/Llama-2-70b-hf"
                ],
                "Parameters": "70B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 61.05
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 55.87
            },
            "User Behavior Alignment": {
                "Overall": 43.24
            },
            "Multi-lingual Abilities": {
                "Overall": 47.85
            }
        },
        "Mixtral-8x7B": {
            "META": {
                "Method": [
                    "Mixtral-8x7B",
                    "https://huggingface.co/mistralai/Mixtral-8x7B-v0.1"
                ],
                "Parameters": "46.7B",
                "Org": "MistralAI",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 59.43
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 54.32
            },
            "User Behavior Alignment": {
                "Overall": 55.31
            },
            "Multi-lingual Abilities": {
                "Overall": 44.69
            }
        },
        "QWen1.5-14B": {
            "META": {
                "Method": [
                    "QWen1.5-14B",
                    "https://huggingface.co/Qwen/Qwen1.5-14B"
                ],
                "Parameters": "14B",
                "Org": "Alibaba",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 67.22
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 60.92
            },
            "User Behavior Alignment": {
                "Overall": 54.92
            },
            "Multi-lingual Abilities": {
                "Overall": 55.21
            }
        },
        "eCeLLM-L": {
            "META": {
                "Method": [
                    "eCeLLM-L",
                    "https://huggingface.co/NingLab/eCeLLM-L"
                ],
                "Parameters": "13B",
                "Org": "OSU NingLab",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 61.54
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 54.84
            },
            "User Behavior Alignment": {
                "Overall": 54.55
            },
            "Multi-lingual Abilities": {
                "Overall": 59.64
            }
        },
        "Vicuna-13B-v1.5": {
            "META": {
                "Method": [
                    "Vicuna-13B-v1.5",
                    "https://huggingface.co/lmsys/vicuna-13b-v1.5"
                ],
                "Parameters": "13B",
                "Org": "LMSys",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 59.64
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 52.63
            },
            "User Behavior Alignment": {
                "Overall": 49.81
            },
            "Multi-lingual Abilities": {
                "Overall": 49.64
            }
        },
        "LLaMA2-13B-Chat": {
            "META": {
                "Method": [
                    "LLaMA2-13B-Chat",
                    "https://huggingface.co/meta-llama/Llama-2-13b-chat-hf"
                ],
                "Parameters": "13B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 51.79
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 45.01
            },
            "User Behavior Alignment": {
                "Overall": 39.95
            },
            "Multi-lingual Abilities": {
                "Overall": 42.99
            }
        },
        "LLaMA2-13B": {
            "META": {
                "Method": [
                    "LLaMA2-13B",
                    "https://huggingface.co/meta-llama/Llama-2-13b-hf"
                ],
                "Parameters": "13B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 45.86
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 39.47
            },
            "User Behavior Alignment": {
                "Overall": 39.43
            },
            "Multi-lingual Abilities": {
                "Overall": 44.23
            }
        },
        "LLaMA3-8B-Instruct": {
            "META": {
                "Method": [
                    "LLaMA3-8B-Instruct",
                    "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct"
                ],
                "Parameters": "8B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 65.26
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 56.84
            },
            "User Behavior Alignment": {
                "Overall": 54.88
            },
            "Multi-lingual Abilities": {
                "Overall": 55.37
            }
        },
        "LLaMA3-8B": {
            "META": {
                "Method": [
                    "LLaMA3-8B",
                    "https://huggingface.co/meta-llama/Meta-Llama-3-8B"
                ],
                "Parameters": "8B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 58.02
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 49.74
            },
            "User Behavior Alignment": {
                "Overall": 44.16
            },
            "Multi-lingual Abilities": {
                "Overall": 51.03
            }
        },
        "QWen1.5-7B": {
            "META": {
                "Method": [
                    "QWen1.5-7B",
                    "https://huggingface.co/Qwen/Qwen1.5-7B"
                ],
                "Parameters": "7B",
                "Org": "Alibaba",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 58.89
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 52.34
            },
            "User Behavior Alignment": {
                "Overall": 49.81
            },
            "Multi-lingual Abilities": {
                "Overall": 50.14
            }
        },
        "eCeLLM-M": {
            "META": {
                "Method": [
                    "eCeLLM-M",
                    "https://huggingface.co/NingLab/eCeLLM-M"
                ],
                "Parameters": "7B",
                "Org": "OSU NingLab",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 63.29
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 48.94
            },
            "User Behavior Alignment": {
                "Overall": 53.78
            },
            "Multi-lingual Abilities": {
                "Overall": 56.08
            }
        },
        "Zephyr-Beta": {
            "META": {
                "Method": [
                    "Zephyr-Beta",
                    "https://huggingface.co/HuggingFaceH4/zephyr-7b-beta"
                ],
                "Parameters": "7B",
                "Org": "HuggingFace H4",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 61.65
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 52.57
            },
            "User Behavior Alignment": {
                "Overall": 44.73
            },
            "Multi-lingual Abilities": {
                "Overall": 45.35
            }
        },
        "Mistral-7B-Instruct": {
            "META": {
                "Method": [
                    "Mistral-7B-Instruct",
                    "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2"
                ],
                "Parameters": "7B",
                "Org": "MistralAI",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 62.03
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 46.36
            },
            "User Behavior Alignment": {
                "Overall": 42.21
            },
            "Multi-lingual Abilities": {
                "Overall": 43.32
            }
        },
        "Mistral-7B": {
            "META": {
                "Method": [
                    "Mistral-7B",
                    "https://huggingface.co/mistralai/Mistral-7B-v0.1"
                ],
                "Parameters": "7B",
                "Org": "MistralAI",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 55.82
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 46.69
            },
            "User Behavior Alignment": {
                "Overall": 46.27
            },
            "Multi-lingual Abilities": {
                "Overall": 41.47
            }
        },
        "Vicuna-7B-v1.5": {
            "META": {
                "Method": [
                    "Vicuna-7B-v1.5",
                    "https://huggingface.co/lmsys/vicuna-7b-v1.5"
                ],
                "Parameters": "7B",
                "Org": "LMSys",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 53.46
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 45.06
            },
            "User Behavior Alignment": {
                "Overall": 41.11
            },
            "Multi-lingual Abilities": {
                "Overall": 43.82
            }
        },
        "LLaMA2-7B-Chat": {
            "META": {
                "Method": [
                    "LLaMA2-7B-Chat",
                    "https://huggingface.co/meta-llama/Llama-2-7b-chat-hf"
                ],
                "Parameters": "7B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 51.67
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 43.48
            },
            "User Behavior Alignment": {
                "Overall": 41.42
            },
            "Multi-lingual Abilities": {
                "Overall": 40.43
            }
        },
        "LLaMA2-7B": {
            "META": {
                "Method": [
                    "LLaMA2-7B",
                    "https://huggingface.co/meta-llama/Llama-2-7b-hf"
                ],
                "Parameters": "7B",
                "Org": "Meta",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 38.22
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 32.81
            },
            "User Behavior Alignment": {
                "Overall": 32.56
            },
            "Multi-lingual Abilities": {
                "Overall": 27.71
            }
        },
        "QWen1.5-4B": {
            "META": {
                "Method": [
                    "QWen1.5-4B",
                    "https://huggingface.co/Qwen/Qwen1.5-4B"
                ],
                "Parameters": "4B",
                "Org": "Alibaba",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 57.21
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 52.56
            },
            "User Behavior Alignment": {
                "Overall": 42.74
            },
            "Multi-lingual Abilities": {
                "Overall": 49.78
            }
        },
        "Phi-2": {
            "META": {
                "Method": [
                    "Phi-2",
                    "https://huggingface.co/microsoft/phi-2"
                ],
                "Parameters": "2.8B",
                "Org": "Microsoft",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 49.34
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 42.83
            },
            "User Behavior Alignment": {
                "Overall": 36.38
            },
            "Multi-lingual Abilities": {
                "Overall": 32.91
            }
        },
        "eCeLLM-S": {
            "META": {
                "Method": [
                    "eCeLLM-S",
                    "https://huggingface.co/NingLab/eCeLLM-S"
                ],
                "Parameters": "2.8B",
                "Org": "OSU NingLab",
                "OpenSource": "Yes",
                "Verified": "Yes"
            },
            "Shopping Concept Understanding": {
                "Overall": 49.4
            },
            "Shopping Knowledge Reasoning": {
                "Overall": 39.06
            },
            "User Behavior Alignment": {
                "Overall": 36.33
            },
            "Multi-lingual Abilities": {
                "Overall": 32.79
            }
        }
    }
}