{"3dsrbench":{"id":"3dsrbench","name":"3DSRBench","topic":"Spatial reasoning","description":"3D spatial reasoning benchmark for large multimodal models (height, location, orientation, multi-object relations).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"aa-briefcase":{"id":"aa-briefcase","name":"AA-Briefcase","topic":"Knowledge work","description":"Artificial Analysis benchmark of long-horizon multi-week knowledge-work projects; Elo from rubric and pairwise judging.","url":"https://artificialanalysis.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"elo","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"aa-fdb-backchannel":{"id":"aa-fdb-backchannel","name":"AA Conversational Dynamics: Backchannel","topic":"Full-duplex speech interaction","description":"Artificial Analysis Conversational Dynamics (Full-Duplex-Bench subset): backchannelling while the user speaks (%).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice"]},"aa-fdb-interruption":{"id":"aa-fdb-interruption","name":"AA Conversational Dynamics: Interruption","topic":"Full-duplex speech interaction","description":"Artificial Analysis Conversational Dynamics (Full-Duplex-Bench subset): handling user interruptions (%).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice"]},"aa-fdb-pause-handling":{"id":"aa-fdb-pause-handling","name":"AA Conversational Dynamics: Pause handling","topic":"Full-duplex speech interaction","description":"Artificial Analysis Conversational Dynamics (Full-Duplex-Bench subset): not barging in during user pauses (%).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice"]},"aa-fdb-turn-taking":{"id":"aa-fdb-turn-taking","name":"AA Conversational Dynamics: Turn-taking","topic":"Full-duplex speech interaction","description":"Artificial Analysis Conversational Dynamics (Full-Duplex-Bench subset): smooth turn-taking (%).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice"]},"aa-index":{"id":"aa-index","name":"AA-Index","topic":"Multi-domain QA","description":"Comprehensive QA index across diverse domains.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"aa-lcr":{"id":"aa-lcr","name":"AA-LCR","topic":"Long-context reasoning","description":"A challenging benchmark measuring language models' ability to extract, reason about, and synthesize information from long-form documents ranging from 10k to 100k tokens.","url":"https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR","stars":"","updated_at":"2025-09-08","leaderboard":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"aa-omniscience":{"id":"aa-omniscience","name":"AA-Omniscience","topic":"Knowledge and hallucination","description":"Benchmark measuring factual recall and hallucination across economically relevant domains.","url":"https://artificialanalysis.ai/evaluations/omniscience","stars":"","updated_at":"","leaderboard":"https://artificialanalysis.ai/evaluations/omniscience","github":"","paper":"","modality":"text","metric":"index","higher_is_better":true,"classification":["Knowledge","Hallucination","QA","Text"]},"aa-omniscience-accuracy":{"id":"aa-omniscience-accuracy","name":"AA-Omniscience Accuracy","topic":"Knowledge and hallucination","description":"Share of AA-Omniscience questions answered correctly (accuracy component of the AA-Omniscience Index).","url":"https://artificialanalysis.ai/evaluations/omniscience","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"aa-omniscience-decision-accuracy":{"id":"aa-omniscience-decision-accuracy","name":"AA-Omniscience-Decision Accuracy","topic":"Knowledge and hallucination","description":"Accuracy on 500 private AA-Omniscience questions posed as five options (four answers plus \"I don't know\") for decision models.","url":"https://artificialanalysis.ai/evaluations/aa-omniscience-decision","stars":"","updated_at":"","leaderboard":"https://artificialanalysis.ai/evaluations/aa-omniscience-decision","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"aa-omniscience-decision-non-hallucination":{"id":"aa-omniscience-decision-non-hallucination","name":"AA-Omniscience-Decision Non-Hallucination Rate","topic":"Knowledge and hallucination","description":"Share of AA-Omniscience-Decision questions not answered correctly where the model chose \"I don't know\".","url":"https://artificialanalysis.ai/evaluations/aa-omniscience-decision","stars":"","updated_at":"","leaderboard":"https://artificialanalysis.ai/evaluations/aa-omniscience-decision","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Knowledge","Text"]},"aa-omniscience-non-hallucination":{"id":"aa-omniscience-non-hallucination","name":"AA-Omniscience Non-Hallucination Rate","topic":"Knowledge and hallucination","description":"Share of AA-Omniscience questions not answered incorrectly (abstains instead of hallucinating) among non-correct answers.","url":"https://artificialanalysis.ai/evaluations/omniscience","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Hallucination","Text"]},"aa-omniscience-public":{"id":"aa-omniscience-public","name":"AA-Omniscience-Public","topic":"Knowledge and hallucination","description":"AA-Omniscience Index on the publicly released question subset (index from -100 to 100).","url":"https://artificialanalysis.ai/evaluations/omniscience","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"index","higher_is_better":true,"classification":["Knowledge","Hallucination","QA","Text"]},"aa-omniscience-public-accuracy":{"id":"aa-omniscience-public-accuracy","name":"AA-Omniscience-Public Accuracy","topic":"Knowledge and hallucination","description":"Share of questions answered correctly on the publicly released AA-Omniscience subset.","url":"https://artificialanalysis.ai/evaluations/omniscience","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Hallucination","QA","Text"]},"aa-s2s-ttfa":{"id":"aa-s2s-ttfa","name":"AA Speech-to-Speech Time to First Audio","topic":"Voice latency","description":"Artificial Analysis speech-to-speech time to first audio in seconds (lower is better).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"seconds","higher_is_better":false,"classification":["Audio","Voice","Latency"]},"aa-speech-agent-arena":{"id":"aa-speech-agent-arena","name":"AA Speech Agent Arena","topic":"Voice agent preference","description":"Artificial Analysis Speech Agent Arena: blind pairwise voice-agent conversations, overall Elo (GPT-Realtime-1.5 anchored at 1000).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"elo","higher_is_better":true,"classification":["Audio","Voice","Arena"]},"aa-speech-agent-task-success":{"id":"aa-speech-agent-task-success","name":"AA Speech Agent Task Success","topic":"Voice agents","description":"Artificial Analysis Speech Agent Arena task completion rate (%).","url":"https://artificialanalysis.ai/speech-to-speech","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Agents"]},"acebench":{"id":"acebench","name":"AceBench","topic":"Industry QA","description":"Industry-focused benchmark assessing domain QA and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Industry","Knowledge","QA","Reasoning","Text"]},"acebench-zh":{"id":"acebench-zh","name":"ACEBench (ZH)","topic":"Tool use (Chinese)","description":"Chinese split of ACEBench tool-use evaluation.","url":"https://github.com/ACEBench/ACEBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Chinese","Tool Use","Text"]},"acos-decision":{"id":"acos-decision","name":"ACOS (decision)","topic":"Aspect sentiment","description":"Aspect-category-opinion-sentiment decisions; per-review F1. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/NUSTM/ACOS","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Sentiment","Decision","Text"]},"acp-bench-bool":{"id":"acp-bench-bool","name":"ACP-Bench Bool","topic":"Safety evaluation (boolean)","description":"Safety and behavior evaluation with yes/no questions.","url":"https://ibm.github.io/ACPBench/","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"https://github.com/ibm/ACPBench","paper":"https://arxiv.org/abs/2410.05669","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"acp-bench-mcq":{"id":"acp-bench-mcq","name":"ACP-Bench MCQ","topic":"Safety evaluation (MCQ)","description":"Safety and behavior evaluation with multiple-choice questions.","url":"https://ibm.github.io/ACPBench/","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"https://github.com/ibm/ACPBench","paper":"https://arxiv.org/abs/2410.05669","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"activitynet-qa":{"id":"activitynet-qa","name":"ActivityNet-QA","topic":"Video QA (open-ended)","description":"Open-ended QA on ActivityNet videos; GPT-judged accuracy.","url":"https://github.com/MILVLG/activitynet-qa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1906.02467","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"acva":{"id":"acva","name":"ACVA","topic":"Arabic culture","description":"Arabic Cultural Value Alignment benchmark (true/false statements about Arab culture).","url":"https://huggingface.co/datasets/FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Culture","Multilingual","Text"]},"advancedif":{"id":"advancedif","name":"AdvancedIF","topic":"Instruction following","description":"Meta's AdvancedIF rubric-based benchmark for complex, multi-turn and system-prompt instruction following.","url":"https://arxiv.org/abs/2511.10507","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"aethercode":{"id":"aethercode","name":"AetherCode","topic":"Code generation","description":"Code generation benchmark for diverse coding tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"afrimed-qa":{"id":"afrimed-qa","name":"AfriMed-QA","topic":"Medical QA","description":"Pan-African medical question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"ag-news":{"id":"ag-news","name":"AG News","topic":"Topic classification","description":"AG News four-class news topic classification.","url":"https://huggingface.co/datasets/fancyzhx/ag_news","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Classification","Text"]},"agentbench":{"id":"agentbench","name":"AgentBench","topic":"LLM agents","description":"Multi-environment agent benchmark (OS, DB, KG, puzzles, household, web shopping, web browsing); overall weighted score.","url":"https://github.com/THUDM/AgentBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2308.03688","modality":"text","metric":"score","higher_is_better":true,"classification":["Agents","Text"]},"agentcompany":{"id":"agentcompany","name":"AgentCompany","topic":"Agent reasoning","description":"Company-level agent reasoning and decision-making benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentdojo":{"id":"agentdojo","name":"AgentDojo","topic":"Agent evaluation","description":"Interactive evaluation suite for autonomous agents across tools and tasks.","url":"https://agentdojo.spylab.ai/","stars":"","updated_at":"","leaderboard":"https://agentdojo.spylab.ai/results/","github":"https://github.com/ethz-spylab/agentdojo","paper":"https://arxiv.org/abs/2406.13352","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"agentharm":{"id":"agentharm","name":"AgentHarm","topic":"Safety / agents","description":"Safety evaluation of agent behaviors under adversarial red-teaming scenarios.","url":"https://huggingface.co/datasets/ai-safety-institute/AgentHarm","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2410.09024","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Safety","Text"]},"agentic-coding":{"id":"agentic-coding","name":"Agentic Coding","topic":"Agentic coding","description":"Agentic coding benchmark for autonomous software tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"agentif":{"id":"agentif","name":"AgentIF","topic":"Instruction following","description":"Instruction-following benchmark built from real agentic scenarios with long, complex system prompts and constraints; score is (CSR+ISR)/2.","url":"https://github.com/THU-KEG/AgentIF","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/THU-KEG/AgentIF","paper":"https://arxiv.org/abs/2505.16944","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"agentif-oneday":{"id":"agentif-oneday","name":"AgentIF-OneDay","topic":"Agentic instruction following","description":"Agentic instruction-following benchmark of full-day, multi-step everyday and office tasks for general agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Instruction following","Text"]},"agents-last-exam":{"id":"agents-last-exam","name":"Agents' Last Exam","topic":"Agentic professional work","description":"Long-horizon agentic workflows across 55 professional sub-industries performed on a computer.","url":"https://agents-last-exam.org/","stars":"","updated_at":"","leaderboard":"https://agents-last-exam.org/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agents-last-exam-score":{"id":"agents-last-exam-score","name":"Agents' Last Exam (average score)","topic":"Agentic professional work","description":"Agents' Last Exam (ALE-CLI, 105-task Linux split) average rubric score (as opposed to pass rate).","url":"https://agents-last-exam.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"agentworldbench-android":{"id":"agentworldbench-android","name":"AgentWorldBench (Android)","topic":"Language world models","description":"AgentWorldBench Android domain: open-ended evaluation of language world models predicting next environment state in Android agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentworldbench-mcp":{"id":"agentworldbench-mcp","name":"AgentWorldBench (MCP)","topic":"Language world models","description":"AgentWorldBench MCP domain: open-ended evaluation of language world models predicting next environment state in MCP agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentworldbench-os":{"id":"agentworldbench-os","name":"AgentWorldBench (OS)","topic":"Language world models","description":"AgentWorldBench OS domain: open-ended evaluation of language world models predicting next environment state in OS agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentworldbench-search":{"id":"agentworldbench-search","name":"AgentWorldBench (Search)","topic":"Language world models","description":"AgentWorldBench Search domain: open-ended evaluation of language world models predicting next environment state in Search agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentworldbench-swe":{"id":"agentworldbench-swe","name":"AgentWorldBench (SWE)","topic":"Language world models","description":"AgentWorldBench SWE domain: open-ended evaluation of language world models predicting next environment state in SWE agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentworldbench-terminal":{"id":"agentworldbench-terminal","name":"AgentWorldBench (Terminal)","topic":"Language world models","description":"AgentWorldBench Terminal domain: open-ended evaluation of language world models predicting next environment state in Terminal agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agentworldbench-web":{"id":"agentworldbench-web","name":"AgentWorldBench (Web)","topic":"Language world models","description":"AgentWorldBench Web domain: open-ended evaluation of language world models predicting next environment state in Web agent interactions; 5-dimension rubric mean normalized 0-100.","url":"https://huggingface.co/datasets/Qwen/AgentWorldBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"agieval":{"id":"agieval","name":"AGIEval (English)","topic":"Exams","description":"English subset of AGIEval; academic and professional exam questions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Text"]},"agieval-lsat-ar":{"id":"agieval-lsat-ar","name":"AGIEval LSAT-AR","topic":"Law exam reasoning","description":"LSAT Analytical Reasoning subset from AGIEval benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Reasoning","Text"]},"ai2d":{"id":"ai2d","name":"AI2D","topic":"Diagram understanding (VQA)","description":"Visual question answering over science and diagram images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Knowledge","Multimodal","QA","STEM","Science","Vision"]},"aicodeking-non-agentic":{"id":"aicodeking-non-agentic","name":"AICodeKing Non-Agentic","topic":"Code generation (non-agentic)","description":"Non-agentic code generation benchmark from AICodeKing.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"aider-code-editing":{"id":"aider-code-editing","name":"Aider Code Editing","topic":"Code editing","description":"Measures interactive code editing quality within the Aider assistant workflow.","url":"https://aider.chat/docs/leaderboards/","stars":"","updated_at":"2025-10-03","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Editing","Text"]},"aider-polyglot":{"id":"aider-polyglot","name":"Aider-Polyglot","topic":"Code assistant eval","description":"Aider polyglot coding leaderboard.","url":"https://aider.chat/docs/leaderboards/","stars":"","updated_at":"2025-09-09","leaderboard":"https://aider.chat/docs/leaderboards/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"aider-polyglot-diff":{"id":"aider-polyglot-diff","name":"Aider-Polyglot (Diff)","topic":"Code assistant eval","description":"Aider polyglot leaderboard using diff mode (pass@2).","url":"https://aider.chat/docs/leaderboards/","stars":"","updated_at":"","leaderboard":"https://aider.chat/docs/leaderboards/","github":"","paper":"","modality":"text","metric":"pass@2","higher_is_better":true,"classification":["Code","Text"]},"aime-2024":{"id":"aime-2024","name":"AIME 2024","topic":"Math (competition)","description":"American Invitational Mathematics Examination 2024 problems.","url":"https://artofproblemsolving.com/wiki/index.php/2024_AIME_I_Problems","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"aime-2024-ko":{"id":"aime-2024-ko","name":"AIME 2024-Ko","topic":"Math (competition, Korean)","description":"Korean translation of AIME 2024 problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Korean","STEM","Text"]},"aime-2025":{"id":"aime-2025","name":"AIME 2025","topic":"Math (competition)","description":"American Invitational Mathematics Examination 2025 problems.","url":"https://artofproblemsolving.com/wiki/index.php/AIME_Problems_and_Solutions","stars":"","updated_at":"2025-09-09","leaderboard":"https://www.kaggle.com/benchmarks/open-benchmarks/aime-2025","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"aime-2026-i":{"id":"aime-2026-i","name":"AIME 2026 I","topic":"Math (competition)","description":"American Invitational Mathematics Examination 2026 I problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"aime-2026-i-with-tools":{"id":"aime-2026-i-with-tools","name":"AIME 2026 I (with tools)","topic":"Math competition (with tools)","description":"AIME 2026 I problems solved with tool (code execution) access.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"ainstein-swe-bench":{"id":"ainstein-swe-bench","name":"AInstein-SWE-Bench","topic":"Agentic coding","description":"AInstein agent coding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"alignbench":{"id":"alignbench","name":"AlignBench","topic":"Alignment and instruction following","description":"Benchmark for instruction-following quality and alignment behavior.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"score","higher_is_better":true,"classification":["Alignment","Instruction Following","Text"]},"all-angles-bench":{"id":"all-angles-bench","name":"All-Angles Bench","topic":"Spatial perception","description":"All-Angles benchmark for spatial recognition and 3D perception.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"almieyar":{"id":"almieyar","name":"Almieyar","topic":"Arabic linguistics","description":"Arabic linguistic competence benchmark across phonology, morphology, syntax, semantics and pragmatics.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Language","Multilingual","Text"]},"alpacaeval":{"id":"alpacaeval","name":"AlpacaEval","topic":"Instruction following","description":"Automatic eval using GPT-4 as a judge.","url":"https://github.com/tatsu-lab/alpaca_eval","stars":"1849","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"alpacaeval-1.0":{"id":"alpacaeval-1.0","name":"AlpacaEval 1.0","topic":"Instruction following","description":"Original AlpacaEval: win rate against text-davinci-003 judged by GPT-4.","url":"https://github.com/tatsu-lab/alpaca_eval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"alpacaeval-2.0":{"id":"alpacaeval-2.0","name":"AlpacaEval 2.0","topic":"Instruction following","description":"Updated AlpacaEval with improved prompts and judging.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"am-2k-mad":{"id":"am-2k-mad","name":"AM-2K (Box) Matting (MAD)","topic":"Image matting","description":"AM-2K (Box): box-prompted animal matting; mean absolute difference x10^3 (lower is better).","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"mad","higher_is_better":false,"classification":["Vision","Multimodal","Perception"]},"amazon-esci":{"id":"amazon-esci","name":"Amazon ESCI","topic":"Product search relevance","description":"Shopping query/product relevance (Exact, Substitute, Complement, Irrelevant); macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/amazon-science/esci-data","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Retrieval","Classification","Text"]},"amc-23":{"id":"amc-23","name":"AMC-23","topic":"Math (competition)","description":"American Mathematics Competition 2023 evaluation.","url":"https://www.maa.org/math-competitions/amc-1012","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"amobench":{"id":"amobench","name":"AMO-Bench","topic":"Math (competition)","description":"Advanced math olympiad-style benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"amobench-zh":{"id":"amobench-zh","name":"AMO-Bench CH","topic":"Math (competition)","description":"Chinese subset of AMO-Bench.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Math","STEM","Text"]},"android-daily":{"id":"android-daily","name":"Android Daily","topic":"GUI agents","description":"Benchmark of long-horizon daily phone-use tasks across multiple Android apps.","url":"https://arxiv.org/abs/2605.27761","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2605.27761","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"androidbench":{"id":"androidbench","name":"AndroidBench","topic":"Software engineering (Android)","description":"Agentic Android app development benchmark; Qwen reports avg@3 on the 95-task public subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"androidworld":{"id":"androidworld","name":"AndroidWorld","topic":"Mobile agents","description":"Benchmark for agents operating Android apps via UI automation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Mobile","Vision"]},"anli":{"id":"anli","name":"ANLI","topic":"Natural language inference","description":"Adversarial NLI (entailment / neutral / contradiction); macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://huggingface.co/datasets/facebook/anli","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"https://arxiv.org/abs/1910.14599","modality":"text","metric":"f1","higher_is_better":true,"classification":["NLI","Text"]},"apex-agents":{"id":"apex-agents","name":"APEX-Agents","topic":"Long horizon professional tasks","description":"APEX benchmark evaluating agents on long-horizon professional tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"apex-agents-v1.1":{"id":"apex-agents-v1.1","name":"APEX-Agents 1.1","topic":"Long horizon professional tasks","description":"Mercor APEX-Agents 1.1 revision: 240 long-horizon cross-application professional-services tasks (investment banking, consulting, corporate law) with updated task specs and judge; Pass@1. Not comparable to APEX-Agents v1.0.","url":"https://huggingface.co/datasets/mercor/apex-agents-v1.1","stars":"","updated_at":"","leaderboard":"https://mercor.com/apex/","github":"","paper":"https://arxiv.org/abs/2601.14242","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"apex-swe":{"id":"apex-swe","name":"APEX-SWE","topic":"Software engineering","description":"Mercor/Cognition benchmark of integration (shipping end-to-end systems on cloud/business/infra services) and observability (debugging production-style failures from telemetry) tasks; Pass@1.","url":"https://www.mercor.com/blog/introducing-apex-swe/","stars":"","updated_at":"","leaderboard":"https://www.mercor.com/blog/introducing-apex-swe/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"api-bank":{"id":"api-bank","name":"API-Bank","topic":"Tool use","description":"API-Bank tool-use benchmark.","url":"https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Text"]},"api-bank-decision":{"id":"api-bank-decision","name":"API-Bank (decision)","topic":"Tool use","description":"API-Bank cases answered as option choices; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tool Use","Decision","Text"]},"arabicmmlu":{"id":"arabicmmlu","name":"ArabicMMLU","topic":"Arabic knowledge","description":"Native Arabic multi-task knowledge benchmark (school exams across MENA countries), 40 tasks.","url":"https://huggingface.co/datasets/MBZUAI/ArabicMMLU","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Text"]},"arc-agi-1":{"id":"arc-agi-1","name":"ARC-AGI-1","topic":"General reasoning","description":"ARC-AGI Phase 1 aggregate accuracy.","url":"https://arcprize.org/","stars":"","updated_at":"","leaderboard":"https://arcprize.org/leaderboard","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"arc-agi-2":{"id":"arc-agi-2","name":"ARC-AGI-2","topic":"General reasoning","description":"ARC-AGI Phase 2 aggregate accuracy.","url":"https://arcprize.org/","stars":"","updated_at":"","leaderboard":"https://arcprize.org/leaderboard","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"arc-agi-3":{"id":"arc-agi-3","name":"ARC-AGI-3","topic":"General reasoning","description":"ARC-AGI-3 interactive reasoning benchmark; Relative Human Action Efficiency (RHAE) score.","url":"https://arcprize.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Reasoning","Agents"]},"arc-average":{"id":"arc-average","name":"ARC Average","topic":"Science QA (average)","description":"Average accuracy across ARC-Easy and ARC-Challenge.","url":"https://github.com/allenai/ai2-arc","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/allenai/ai2-arc","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text"]},"arc-cf":{"id":"arc-cf","name":"ARC-CF","topic":"Science QA (few-shot)","description":"AI2 Reasoning Challenge (Challenge set), few-shot configuration.","url":"https://github.com/allenai/ai2-arc","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text"]},"arc-challenge":{"id":"arc-challenge","name":"ARC-Challenge","topic":"Science QA","description":"Hard subset of AI2 Reasoning Challenge; grade-school science.","url":"https://github.com/allenai/ai2-arc","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text"]},"arc-challenge-chat":{"id":"arc-challenge-chat","name":"ARC-Challenge Chat","topic":"Science QA (chat)","description":"AI2 Reasoning Challenge (Challenge set) evaluated in chat/free-form mode.","url":"https://github.com/allenai/ai2-arc","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text"]},"arc-challenge-de":{"id":"arc-challenge-de","name":"ARC-Challenge (DE)","topic":"Science QA (German)","description":"German translation of the ARC Challenge benchmark.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"norm_log_acc","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text","German"]},"arc-challenge-multilingual":{"id":"arc-challenge-multilingual","name":"ARC-Challenge Multilingual","topic":"Science QA (multilingual)","description":"Multilingual variant of the ARC Challenge science QA benchmark.","url":"https://github.com/allenai/ai2-arc","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","STEM","Science","Text"]},"arc-easy":{"id":"arc-easy","name":"ARC-Easy","topic":"Science QA","description":"Easier subset of AI2 Reasoning Challenge.","url":"https://github.com/allenai/ai2-arc","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text"]},"arc-easy-de":{"id":"arc-easy-de","name":"ARC-Easy (DE)","topic":"Science QA (German)","description":"German translation of the ARC Easy science QA benchmark.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"norm_log_acc","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text","German"]},"arena-hard":{"id":"arena-hard","name":"Arena-Hard","topic":"Chat ability","description":"Hard prompts on Chatbot Arena.","url":"https://github.com/lm-sys/arena-hard-auto","stars":"920","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"arena-hard-v2":{"id":"arena-hard-v2","name":"Arena-Hard V2","topic":"Chat ability","description":"Updated Arena-Hard v2 prompts on Chatbot Arena.","url":"https://github.com/lm-sys/arena-hard-auto","stars":"920","updated_at":"2025-08-30","leaderboard":"","github":"https://github.com/lmarena/arena-hard-auto","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"arena-hard-v2-creative-writing":{"id":"arena-hard-v2-creative-writing","name":"Arena-Hard V2 Creative Writing","topic":"Creative writing","description":"Chatbot Arena Hard V2 creative writing win-rate subset.","url":"https://github.com/lm-sys/arena-hard-auto","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Creative Writing","Text"]},"arena-hard-v2-hard-prompt":{"id":"arena-hard-v2-hard-prompt","name":"Arena-Hard V2 Hard Prompt","topic":"Chat ability","description":"Chatbot Arena Hard V2 benchmark using the hard prompt win-rate subset.","url":"https://github.com/lm-sys/arena-hard-auto","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"arkitscenes":{"id":"arkitscenes","name":"ARKitScenes","topic":"3D scene understanding","description":"ARKitScenes benchmark for assessing 3D scene reconstruction and understanding from mixed reality captures.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["3D","Multimodal","Vision"]},"art-agent-red-teaming":{"id":"art-agent-red-teaming","name":"ART Agent Red Teaming","topic":"Agent robustness","description":"Evaluation suite for adversarial red-teaming of autonomous AI agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2507.20526","modality":"text","metric":"percent","higher_is_better":false,"classification":["Agents","Safety","Text"]},"artifactsbench":{"id":"artifactsbench","name":"ArtifactsBench","topic":"Agentic coding","description":"Artifacts-focused coding and tool-use benchmark evaluating generated code artifacts.","url":"","stars":"","updated_at":"2025-10-03","leaderboard":"https://artifactsbenchmark.github.io/leaderboard.html","github":"https://github.com/Tencent-Hunyuan/ArtifactsBenchmark","paper":"https://arxiv.org/abs/2507.04952","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"arxivmath":{"id":"arxivmath","name":"ArxivMath","topic":"Research math","description":"Math problems derived from recent arXiv papers (reported in Anthropic Fable 5/Mythos 5 system card).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"asr-ami":{"id":"asr-ami","name":"ASR AMI","topic":"ASR","description":"Automatic speech recognition benchmark on AMI meeting speech.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-commonvoice-8-ja":{"id":"asr-commonvoice-8-ja","name":"ASR Common Voice 8 (Japanese)","topic":"ASR","description":"Japanese automatic speech recognition on the Common Voice 8.0 Japanese test set (character error rate).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"cer","higher_is_better":false,"classification":["ASR","Audio","Japanese"]},"asr-earnings22":{"id":"asr-earnings22","name":"ASR Earnings22","topic":"ASR","description":"Automatic speech recognition benchmark on Earnings22 financial calls.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-gigaspeech":{"id":"asr-gigaspeech","name":"ASR GigaSpeech","topic":"ASR","description":"Automatic speech recognition benchmark on GigaSpeech.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-jsut-basic5000":{"id":"asr-jsut-basic5000","name":"ASR JSUT Basic5000","topic":"ASR","description":"Japanese automatic speech recognition on JSUT basic5000 (character error rate).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"cer","higher_is_better":false,"classification":["ASR","Audio","Japanese"]},"asr-librispeech-clean":{"id":"asr-librispeech-clean","name":"ASR LibriSpeech Clean","topic":"ASR","description":"Automatic speech recognition benchmark on LibriSpeech clean split.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-librispeech-other":{"id":"asr-librispeech-other","name":"ASR LibriSpeech Other","topic":"ASR","description":"Automatic speech recognition benchmark on LibriSpeech other split.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-reazonspeech":{"id":"asr-reazonspeech","name":"ASR ReazonSpeech","topic":"ASR","description":"Japanese automatic speech recognition on the ReazonSpeech held-out test set (character error rate).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"cer","higher_is_better":false,"classification":["ASR","Audio","Japanese"]},"asr-spgispeech":{"id":"asr-spgispeech","name":"ASR SPGISpeech","topic":"ASR","description":"Automatic speech recognition benchmark on SPGISpeech.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-ted-lium":{"id":"asr-ted-lium","name":"ASR TED-LIUM","topic":"ASR","description":"Automatic speech recognition benchmark on TED-LIUM.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"asr-voxpopuli":{"id":"asr-voxpopuli","name":"ASR VoxPopuli","topic":"ASR","description":"Automatic speech recognition benchmark on VoxPopuli.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["ASR","Audio"]},"astabench":{"id":"astabench","name":"AstaBench","topic":"Agent evaluation","description":"Evaluates science agents across literature understanding, data analysis, planning, tool use, coding, and search.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"https://huggingface.co/spaces/allenai/asta-bench-leaderboard","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Reasoning","STEM","Science","Text"]},"attaq":{"id":"attaq","name":"AttaQ","topic":"Safety / jailbreak","description":"Adversarial jailbreak suite measuring refusal robustness against targeted attack prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"audio-multichallenge":{"id":"audio-multichallenge","name":"Audio MultiChallenge","topic":"Spoken multi-turn conversation","description":"Scale AI Audio MultiChallenge: multi-turn spoken conversation benchmark; average pass rate (APR).","url":"https://thinkingmachines.ai/blog/interaction-models/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Chat"]},"autocodebench":{"id":"autocodebench","name":"AutoCodeBench","topic":"Autonomous coding","description":"End-to-end autonomous coding benchmark with unit-test based execution across diverse repositories and tasks.","url":"https://autocodebench.github.io/","stars":"","updated_at":"2025-09-10","leaderboard":"https://autocodebench.github.io/leaderboard.html","github":"https://github.com/Tencent-Hunyuan/AutoCodeBenchmark","paper":"https://arxiv.org/abs/2508.09101","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"autocodebench-lite":{"id":"autocodebench-lite","name":"AutoCodeBench-Lite","topic":"Autonomous coding","description":"Lite version of AutoCodeBench focusing on smaller tasks with the same end-to-end, unit-test-based evaluation.","url":"https://autocodebench.github.io/","stars":"","updated_at":"2025-09-10","leaderboard":"https://autocodebench.github.io/leaderboard.html","github":"https://github.com/Tencent-Hunyuan/AutoCodeBenchmark","paper":"https://arxiv.org/abs/2508.09101","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"autologi":{"id":"autologi","name":"AutoLogi","topic":"Logical reasoning","description":"AutoLogi benchmark evaluating automated logical reasoning accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"automationbench":{"id":"automationbench","name":"AutomationBench","topic":"Agentic business workflows","description":"Zapier benchmark of end-to-end business workflows via REST APIs across 47 simulated apps; pass/fail on deterministic state assertions.","url":"https://arxiv.org/abs/2604.18934","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"automationbench-aa":{"id":"automationbench-aa","name":"AutomationBench-AA","topic":"Business workflows","description":"Artificial Analysis run of AutomationBench end-to-end business workflows.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Agents"},"automationbench-partial":{"id":"automationbench-partial","name":"AutomationBench (partial credit)","topic":"Business workflows","description":"AutomationBench end-to-end business workflows scored with partial credit per task.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"babe":{"id":"babe","name":"BABE","topic":"STEM reasoning","description":"STEM reasoning benchmark evaluating broad applied and basic engineering knowledge.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["STEM","Reasoning","Text"]},"babyvision":{"id":"babyvision","name":"BabyVision","topic":"Visual reasoning","description":"Visual reasoning benchmark testing basic visual perception and understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Reasoning","Vision"]},"babyvision-with-tools":{"id":"babyvision-with-tools","name":"BabyVision (With Tools)","topic":"Visual reasoning","description":"BabyVision visual perception benchmark evaluated with tool access in an agent harness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Reasoning","Vision"]},"balrog":{"id":"balrog","name":"BALROG","topic":"Agent robustness","description":"Benchmark for assessing LLM agents under adversarial and out-of-distribution tool-use scenarios.","url":"https://balrogai.com/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/balrog-ai/BALROG","paper":"https://arxiv.org/abs/2411.13543","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Safety","Text"]},"bankertoolbench":{"id":"bankertoolbench","name":"BankerToolBench","topic":"Investment banking agent tasks","description":"BankerToolBench: tool-using agent tasks modeled on investment-banking workflows, scored with dataset rubrics.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"banking77":{"id":"banking77","name":"BANKING77","topic":"Intent classification","description":"Bank customer intent among 77 classes; macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/PolyAI-LDN/task-specific-datasets","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"https://arxiv.org/abs/2003.04807","modality":"text","metric":"f1","higher_is_better":true,"classification":["Classification","Text"]},"bbh":{"id":"bbh","name":"BBH","topic":"Multi-task reasoning","description":"Hard subset of BIG-bench with diverse reasoning tasks.","url":"https://github.com/suzgunmirac/BIG-bench-Hard","stars":"510","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2210.09261","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"bbh-decision":{"id":"bbh-decision","name":"BBH (decision)","topic":"Multi-task reasoning","description":"BIG-Bench Hard fixed-option tasks answered directly with no working; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/suzgunmirac/BIG-Bench-Hard","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Decision","Text"]},"bbh-zh":{"id":"bbh-zh","name":"BBH-ZH","topic":"Multi-task reasoning (Chinese)","description":"Chinese translation of BIG-Bench Hard reasoning tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Reasoning","Text"]},"bbl-hard":{"id":"bbl-hard","name":"BBL-Hard","topic":"Multi-task reasoning","description":"Harder subset of Big Bench Lite.","url":"https://github.com/google/BIG-bench","stars":"3110","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"bbq":{"id":"bbq","name":"BBQ","topic":"Bias evaluation","description":"Bias Benchmark for Question Answering evaluating social biases across contexts.","url":"https://huggingface.co/datasets/heegyu/bbq","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/jinjh0123/BBG","paper":"https://arxiv.org/abs/2110.08193","modality":"text","metric":"percent","higher_is_better":true,"classification":["Bias","QA","Ethics","Text"]},"beam-128k":{"id":"beam-128k","name":"BEAM 128K","topic":"Long-context memory","description":"BEAM long-context memory benchmark at 128K tokens.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Long context"},"beavertails":{"id":"beavertails","name":"BeaverTails","topic":"Safety / harmfulness","description":"Safety benchmark evaluating harmfulness in model responses.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"belebele":{"id":"belebele","name":"Belebele","topic":"Multilingual reading comprehension","description":"Parallel multiple-choice reading-comprehension benchmark over FLORES passages in 122 language variants.","url":"https://huggingface.co/datasets/facebook/belebele","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Reading Comprehension","Text"]},"benchcad":{"id":"benchcad","name":"BenchCAD","topic":"CAD generation","description":"CAD modelling benchmark for agents producing 3D CAD artifacts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Vision"]},"benchcad-python-tool":{"id":"benchcad-python-tool","name":"BenchCAD (python tool)","topic":"CAD","description":"BenchCAD with a Python tool available.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Vision"]},"beyondaime":{"id":"beyondaime","name":"BeyondAIME","topic":"Math (beyond AIME)","description":"Advanced math problems exceeding AIME difficulty.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"bfcl":{"id":"bfcl","name":"BFCL","topic":"Code reasoning","description":"Benchmark for functional code correctness and logic.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"bfcl-decision":{"id":"bfcl-decision","name":"BFCL (decision)","topic":"Function calling","description":"Yes/no per supplied function on BFCL cases; case exact accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/gorilla-llm/gorilla","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tool Use","Decision","Text"]},"bfcl-live-v2":{"id":"bfcl-live-v2","name":"BFCL Live v2","topic":"Finance QA","description":"Financial compliance and literacy questions from the BFCL Live v2 benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Knowledge","Text"]},"bfcl-v2":{"id":"bfcl-v2","name":"BFCL v2","topic":"Code reasoning","description":"Second release of the BFCL benchmark focusing on functional code correctness and logic.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"bfcl-v3":{"id":"bfcl-v3","name":"BFCL v3","topic":"Code reasoning","description":"Benchmark for functional code correctness and logic (v3).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"bfcl-v3-ja":{"id":"bfcl-v3-ja","name":"BFCL v3 (Japanese)","topic":"Function calling (Japanese)","description":"Japanese translation of BFCL v3 function-calling benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Text","Japanese"]},"bfcl-v3-live":{"id":"bfcl-v3-live","name":"BFCL v3 (Live)","topic":"Tool calling","description":"BFCL v3 Live subset for real-time tool calling evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Text"]},"bfcl-v3-multi-turn":{"id":"bfcl-v3-multi-turn","name":"BFCL v3 (Multi-Turn)","topic":"Tool calling","description":"BFCL v3 Multi-Turn subset for multi-turn tool calling evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Text"]},"bfcl-v4":{"id":"bfcl-v4","name":"BFCL v4","topic":"Code reasoning","description":"BFCL v4 benchmark for functional code correctness and logic.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"bfcl-v4-multi-turn":{"id":"bfcl-v4-multi-turn","name":"BFCL v4 Multi-Turn","topic":"Function calling","description":"Multi-turn category of the Berkeley Function Calling Leaderboard v4.","url":"https://gorilla.cs.berkeley.edu/leaderboard.html","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Agents","Text"]},"bharatmath":{"id":"bharatmath","name":"BharatMath","topic":"Indic languages","description":"Math word problems in Indian languages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"big-bench":{"id":"big-bench","name":"BIG-Bench","topic":"Multi-task reasoning","description":"BIG-bench overall performance (original).","url":"https://github.com/google/BIG-bench","stars":"3110","updated_at":"","leaderboard":"","github":"https://github.com/google/BIG-bench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"big-bench-audio":{"id":"big-bench-audio","name":"Big Bench Audio","topic":"Speech reasoning","description":"Artificial Analysis Big Bench Audio: spoken-question reasoning benchmark adapted from BIG-Bench Hard.","url":"https://huggingface.co/datasets/ArtificialAnalysis/big_bench_audio","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Reasoning"]},"big-bench-extra-hard":{"id":"big-bench-extra-hard","name":"BIG-Bench Extra Hard","topic":"Multi-task reasoning","description":"Extra hard subset of BIG-bench tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"big-finance-bench":{"id":"big-finance-bench","name":"Big Finance Bench","topic":"Finance","description":"Finance benchmark reported in the GPT-5.6 announcement.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"bigcodebench":{"id":"bigcodebench","name":"BigCodeBench","topic":"Code Generation","description":"BigCodeBench evaluates large language models on practical code generation tasks with unit-test verification.","url":"https://huggingface.co/datasets/bigcode/bigcodebench","stars":"","updated_at":"2025-09-09","leaderboard":"https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard","github":"https://github.com/bigcode-project/BigCodeBench","paper":"https://arxiv.org/abs/2409.00001","modality":"text","metric":"pass@1","higher_is_better":true,"classification":["Code","Engineering","Text"]},"bigcodebench-hard":{"id":"bigcodebench-hard","name":"BigCodeBench Hard","topic":"Code generation (hard)","description":"Harder variant of BigCodeBench testing complex programming and library tasks with function-level code generation.","url":"https://bigcode-project.github.io/bigcodebench/","stars":"","updated_at":"2025-09-22","leaderboard":"https://bigcode-project.github.io/bigcodebench/","github":"https://github.com/bigcode-project/bigcodebench","paper":"","modality":"text","metric":"pass@1","higher_is_better":true,"classification":["Code","Text"]},"biobench":{"id":"biobench","name":"BIOBench","topic":"Biology reasoning","description":"Biology knowledge and reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","STEM","Text"]},"biology-instruction":{"id":"biology-instruction","name":"Biology-Instruction","topic":"Biology multi-omics","description":"Multi-omics sequence reasoning benchmark for biological data understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Biology","Science","STEM","Vision"]},"biolp-bench":{"id":"biolp-bench","name":"BioLP-Bench","topic":"Biomedical NLP","description":"Comprehensive biomedical language processing benchmark evaluating LLMs across tasks like NER, relation extraction, and QA.","url":"https://github.com/baceolus/BioLP-bench","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/baceolus/BioLP-bench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"biomysterybench":{"id":"biomysterybench","name":"BioMysteryBench","topic":"Biology research","description":"Anthropic biology research benchmark of open-ended data-analysis 'mysteries'; human-solved subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","Agents","Reasoning","Text"]},"biomysterybench-hard":{"id":"biomysterybench-hard","name":"BioMysteryBench (Hard)","topic":"Biology research","description":"BioMysteryBench hard subset (problems not solved by human experts).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","Agents","Reasoning","Text"]},"bioreason-coding":{"id":"bioreason-coding","name":"BioReason Coding","topic":"Genomics","description":"Coding variant-effect prediction from DNA (balanced accuracy).","url":"https://huggingface.co/HuggingFaceBio/Carbon-X","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","STEM","Text"]},"bioreason-non-snv":{"id":"bioreason-non-snv","name":"BioReason Non-SNV","topic":"Genomics","description":"Complex (non-SNV) variant-effect prediction from DNA (balanced accuracy).","url":"https://huggingface.co/HuggingFaceBio/Carbon-X","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","STEM","Text"]},"bird-sql":{"id":"bird-sql","name":"Bird-SQL","topic":"Text-to-SQL","description":"Natural language to SQL generation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"bixbench":{"id":"bixbench","name":"BixBench","topic":"Bioinformatics","description":"Benchmark of real-world bioinformatics and data-analysis tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Agents","Code","Text"]},"blend":{"id":"blend","name":"BLEnD","topic":"Cultural understanding","description":"Benchmark assessing cultural knowledge, sensitivity, and language use across contexts.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"blink":{"id":"blink","name":"BLINK","topic":"Multimodal grounding","description":"Evaluates visual-language grounding and reference resolution to reduce hallucinations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Grounding","Multimodal","Vision"]},"blueprint-bench-2":{"id":"blueprint-bench-2","name":"Blueprint-Bench 2","topic":"Spatial reasoning","description":"Andon Labs agentic spatial-reasoning benchmark: reconstruct 2D floor plans from ~20 interior photos per apartment (50 apartments).","url":"https://arxiv.org/abs/2509.25229","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","Multimodal","Reasoning","Vision"]},"bob-hvr":{"id":"bob-hvr","name":"BoB-HVR","topic":"Composite capability index","description":"Hard, Versatile, and Relevant composite score across eight capability buckets.","url":"","stars":"","updated_at":"2024-07-26","leaderboard":"","github":"","paper":"","modality":"text","metric":"index (0-100)","higher_is_better":true,"classification":["Composite","Meta","Text"]},"bold":{"id":"bold","name":"BOLD","topic":"Bias evaluation","description":"Bias in Open-ended Language Dataset probing demographic biases in text generation.","url":"https://huggingface.co/datasets/AmazonScience/bold","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/amazon-science/bold","paper":"https://arxiv.org/abs/2101.11718","modality":"text","metric":"score","higher_is_better":false,"classification":["Bias","Generation","Ethics","Text"]},"boolq":{"id":"boolq","name":"BoolQ","topic":"Reading comprehension","description":"Yes/no QA from naturally occurring questions.","url":"https://github.com/google-research-datasets/boolean-questions","stars":"171","updated_at":"","leaderboard":"https://llm-evals.formula-labs.com/","github":"https://github.com/google-research-datasets/boolean-questions","paper":"https://arxiv.org/pdf/1905.10044","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"borda-count-multilingual":{"id":"borda-count-multilingual","name":"Borda Count (Multilingual)","topic":"Aggregate ranking","description":"Borda count aggregate ranking across multilingual benchmarks; lower is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"borda_count","higher_is_better":false,"classification":["Ranking","Multilingual","Text"]},"bpomp-decision":{"id":"bpomp-decision","name":"BPoMP (decision)","topic":"Preference prediction","description":"BPoMP preference decisions; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/spaces/multimodalart/jev-decision-index","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Preference","Decision","Text"]},"bridgebench":{"id":"bridgebench","name":"BridgeBench","topic":"Reasoning","description":"BridgeBench evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"bright-decision":{"id":"bright-decision","name":"BRIGHT (decision)","topic":"Reasoning-intensive retrieval","description":"Rate candidate documents for BRIGHT queries; nDCG@10. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://brightbenchmark.github.io/","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"ndcg@10","higher_is_better":true,"classification":["Retrieval","Text"]},"brokenarxiv":{"id":"brokenarxiv","name":"BrokenArXiv","topic":"Research math","description":"Benchmark of subtly broken/false statements derived from recent arXiv math papers; models must detect and prove or refute them.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"browsecomp":{"id":"browsecomp","name":"BrowseComp","topic":"Web browsing","description":"Web browsing comprehension and competence benchmark.","url":"https://openai.com/index/browsecomp/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text","Web"]},"browsecomp-agent-swarm":{"id":"browsecomp-agent-swarm","name":"BrowseComp (Agent Swarm)","topic":"Web browsing agents","description":"BrowseComp evaluated with Moonshot's Agent Swarm multi-agent orchestration (parallel sub-agents) instead of a single agent.","url":"https://openai.com/index/browsecomp/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Search","Web","Text"]},"browsecomp-plus":{"id":"browsecomp-plus","name":"BrowseComp-Plus","topic":"Deep research","description":"BrowseComp-Plus: BrowseComp questions over a fixed, curated document corpus for reproducible deep-research agent evaluation.","url":"https://github.com/texttron/BrowseComp-Plus","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Search","Text"]},"browsecomp-vl":{"id":"browsecomp-vl","name":"BrowseComp-VL","topic":"Multimodal browsing","description":"Vision-language variant of BrowseComp requiring multimodal web browsing.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Web","Multimodal","Vision"]},"browsecomp-with-content-manager":{"id":"browsecomp-with-content-manager","name":"BrowseComp (With Content Manager)","topic":"Web browsing","description":"BrowseComp benchmark evaluated with content manager assistance.","url":"https://openai.com/index/browsecomp/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text","Web","Tools"]},"browsecomp-zh":{"id":"browsecomp-zh","name":"BrowseComp_zh","topic":"Web browsing (Chinese)","description":"Chinese variant of the BrowseComp web browsing benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Text","Web"]},"browsecomp-zh-with-content-manager":{"id":"browsecomp-zh-with-content-manager","name":"BrowseComp-ZH (With Content Manager)","topic":"Web browsing (Chinese)","description":"Chinese BrowseComp benchmark evaluated with content manager assistance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Text","Web","Tools"]},"brumo25":{"id":"brumo25","name":"BRuMo25","topic":"Math competition","description":"BruMo 2025 olympiad-style mathematics benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Olympiad","Text"]},"business-arena":{"id":"business-arena","name":"Business Arena","topic":"Business simulation","description":"Simulated business-operation arena; final profit/value in USD achieved by the agent.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"usd","higher_is_better":true,"classification":["Agents","Text"]},"buzzbench":{"id":"buzzbench","name":"BuzzBench","topic":"Humor analysis","description":"A humour analysis benchmark.","url":"","stars":"","updated_at":"","leaderboard":"https://eqbench.com/buzzbench.html","github":"https://github.com/EQ-bench","paper":"https://arxiv.org/abs/2312.06281","modality":"text","metric":"percent","higher_is_better":true,"classification":["Humor","Text"]},"c-eval":{"id":"c-eval","name":"C-Eval","topic":"Chinese exams","description":"Chinese college-level exam benchmark.","url":"https://github.com/SJTU-LIT/ceval","stars":"1768","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Exams","Text"]},"c3":{"id":"c3","name":"C3","topic":"Chinese reading comprehension","description":"Multiple-choice Chinese machine reading comprehension (CLUE C3).","url":"https://dataset.org/c3/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1904.09679","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Reading comprehension","Text"]},"c3-bench":{"id":"c3-bench","name":"C3-Bench","topic":"Reasoning (Chinese)","description":"Comprehensive Chinese reasoning capability benchmark.","url":"https://github.com/Tencent-Hunyuan/C3-Benchmark","stars":"35","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Reasoning","Text"]},"caselaw":{"id":"caselaw","name":"CaseLaw","topic":"Legal QA","description":"Question answering and reasoning over legal case documents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","QA","Text"]},"caselaw-v2":{"id":"caselaw-v2","name":"CaseLaw v2","topic":"Legal reasoning","description":"U.S. case law benchmark evaluating legal reasoning and judgment over court opinions.","url":"https://www.vals.ai/benchmarks/case_law_v2-09-20-2025","stars":"","updated_at":"2025-09-20","leaderboard":"https://www.vals.ai/benchmarks/case_law_v2-09-20-2025","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Law","Reasoning","Text"]},"casereportbench":{"id":"casereportbench","name":"CaseReportBench","topic":"Medical information extraction","description":"Clinical information extraction from medical case reports into structured fields.","url":"https://huggingface.co/datasets/cxyzhang/caseReportBench_ClinicalDenseExtraction_Benchmark","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Extraction","Text"]},"cc-bench-v2-backend":{"id":"cc-bench-v2-backend","name":"CC-Bench-V2 Backend","topic":"Agentic coding","description":"Z.ai CC-Bench-V2 backend development tasks in a Claude Code-style agent harness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"cc-bench-v2-frontend":{"id":"cc-bench-v2-frontend","name":"CC-Bench-V2 Frontend","topic":"Agentic coding","description":"Z.ai CC-Bench-V2 frontend development tasks in a Claude Code-style agent harness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"cc-bench-v2-repo-exploration":{"id":"cc-bench-v2-repo-exploration","name":"CC-Bench-V2 Repo Exploration","topic":"Agentic coding","description":"Z.ai CC-Bench-V2 repository exploration tasks in a Claude Code-style agent harness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"cc-ocr":{"id":"cc-ocr","name":"CC-OCR","topic":"OCR (cross-lingual)","description":"Cross-lingual OCR benchmark evaluating character recognition across mixed-language documents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","Vision"]},"cc-ocr-kie":{"id":"cc-ocr-kie","name":"CC-OCR (KIE)","topic":"Key information extraction","description":"Key information extraction track of the CC-OCR benchmark.","url":"https://github.com/AlibabaResearch/AdvancedLiterateMachinery/tree/main/Benchmarks/CC-OCR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"cc-ocr-multilingual":{"id":"cc-ocr-multilingual","name":"CC-OCR (Multilingual)","topic":"OCR (multilingual)","description":"Multilingual text recognition track of the CC-OCR benchmark.","url":"https://github.com/AlibabaResearch/AdvancedLiterateMachinery/tree/main/Benchmarks/CC-OCR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"ccbench":{"id":"ccbench","name":"CCBench","topic":"Chinese culture VQA","description":"MMBench-family benchmark on Chinese culture (painting, calligraphy, cuisine, history...).","url":"https://mmbench.opencompass.org.cn/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2307.06281","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Multimodal","Vision"]},"ccpm":{"id":"ccpm","name":"CCPM","topic":"Chinese classical poetry","description":"Chinese Classical Poetry Matching: match modern Chinese to classical poetry lines.","url":"https://github.com/THUNLP-AIPoet/CCPM","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2106.01979","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Language","Text"]},"cfcolor-decision":{"id":"cfcolor-decision","name":"cfcolor (decision)","topic":"Color perception","description":"Color-name / color judgement decisions; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/spaces/multimodalart/jev-decision-index","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Perception","Decision","Text"]},"cfeval":{"id":"cfeval","name":"CFEval","topic":"Coding ELO / contest eval","description":"Contest-style coding evaluation with ELO-like scoring.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"ELO","higher_is_better":true,"classification":["Code","Text"]},"cgbench":{"id":"cgbench","name":"CGBench","topic":"Long video QA","description":"Cartoon/CG long video question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","QA","Vision"]},"charades-sta":{"id":"charades-sta","name":"Charades-STA","topic":"Video grounding","description":"Charades-STA temporal grounding (mIoU).","url":"https://huggingface.co/datasets/VLM2Vec/Charades-STA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"chartbench":{"id":"chartbench","name":"ChartBench","topic":"Chart understanding","description":"Chart comprehension benchmark covering annotated and unannotated chart types.","url":"https://chartbench.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chart","Multimodal","QA","Vision"]},"chartmuseum":{"id":"chartmuseum","name":"ChartMuseum","topic":"Chart understanding","description":"Large-scale curated collection of charts for evaluating parsing, grounding, and reasoning.","url":"https://chartmuseum-leaderboard.github.io/","stars":"","updated_at":"","leaderboard":"https://chartmuseum-leaderboard.github.io/","github":"https://github.com/Liyan06/ChartMuseum","paper":"https://arxiv.org/abs/2505.13444","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Grounding","Multimodal","Reasoning","Vision"]},"chartography":{"id":"chartography","name":"Chartography","topic":"Chart understanding","description":"Surge AI benchmark for professional chart understanding (no tools).","url":"https://surgehq.ai/blog/chartography","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"chartography-with-tools":{"id":"chartography-with-tools","name":"Chartography (With Tools)","topic":"Chart understanding","description":"Surge AI Chartography evaluated with tool (code execution) access.","url":"https://surgehq.ai/blog/chartography","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision","Agents"]},"chartqa":{"id":"chartqa","name":"ChartQA","topic":"Chart understanding (VQA)","description":"Visual question answering over charts and plots.","url":"https://github.com/vis-nlp/ChartQA","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Knowledge","Multimodal","QA","Vision"]},"chartqa-pro":{"id":"chartqa-pro","name":"ChartQA-Pro","topic":"Chart understanding (VQA)","description":"Professional-grade chart question answering with diverse chart types and complex reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Knowledge","Multimodal","QA","Reasoning","Vision"]},"chartqa-pro-with-tools":{"id":"chartqa-pro-with-tools","name":"ChartQA-Pro (with tools)","topic":"Chart understanding (with tools)","description":"ChartQA-Pro evaluated with tool access.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Multimodal","Reasoning","Tools","Vision"]},"charxiv-dq":{"id":"charxiv-dq","name":"CharXiv (DQ)","topic":"Chart description (PDF)","description":"Scientific chart/table descriptive questions from arXiv PDFs.","url":"https://charxiv.github.io/","stars":"","updated_at":"","leaderboard":"https://charxiv.github.io/","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Multimodal","Vision"]},"charxiv-rq":{"id":"charxiv-rq","name":"CharXiv (RQ)","topic":"Chart reasoning (PDF)","description":"Scientific chart/table reasoning questions from arXiv PDFs.","url":"https://charxiv.github.io/","stars":"","updated_at":"","leaderboard":"https://charxiv.github.io/","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Multimodal","Reasoning","Vision"]},"charxiv-rq-with-tools":{"id":"charxiv-rq-with-tools","name":"CharXiv (RQ, With Tools)","topic":"Chart understanding","description":"CharXiv reasoning questions evaluated with tool access.","url":"https://charxiv.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision","Agents"]},"chessbench-decision":{"id":"chessbench-decision","name":"ChessBench (decision)","topic":"Chess","description":"Choose the best move among candidates (ChessBench); accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/google-deepmind/searchless_chess","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Games","Decision","Text"]},"chexpert":{"id":"chexpert","name":"CheXpert","topic":"Medical imaging","description":"Chest X-ray condition classification, macro F1 over top 5 conditions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"chid":{"id":"chid","name":"CHID","topic":"Chinese idiom cloze","description":"Chinese IDiom cloze-test dataset (CLUE).","url":"https://github.com/chujiezheng/ChID-Dataset","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1906.01265","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Language","Text"]},"chinese-simpleqa":{"id":"chinese-simpleqa","name":"Chinese SimpleQA","topic":"QA (Chinese)","description":"Chinese variant of the SimpleQA benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Knowledge","QA","Text"]},"cibench":{"id":"cibench","name":"CIBench","topic":"Code interpreter","description":"Evaluation of LLMs using a code interpreter for data-science tasks; EN/ZH average.","url":"https://github.com/open-compass/CIBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2407.10499","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Coding","Tool Use"]},"circa":{"id":"circa","name":"Circa","topic":"Indirect answers","description":"Interpreting indirect yes/no answers in dialogue.","url":"https://github.com/google-research-datasets/circa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Language","Text"]},"civil-comments":{"id":"civil-comments","name":"Civil Comments","topic":"Toxicity classification","description":"Toxicity classification of online comments (Jigsaw Civil Comments).","url":"https://huggingface.co/datasets/google/civil_comments","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Classification","Text"]},"cl-bench":{"id":"cl-bench","name":"CL-Bench","topic":"Long-context reasoning","description":"Comprehensive long-context benchmark evaluating reasoning over extended contexts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"cl-bench-life":{"id":"cl-bench-life","name":"CL-bench Life","topic":"Context learning","description":"Tencent Hy context-learning benchmark built from real-life scenarios (companion to CL-bench); models must learn and apply rules given in long, messy contexts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"cladder":{"id":"cladder","name":"CLadder","topic":"Causal reasoning","description":"Causal reasoning questions across the ladder of causation; accuracy (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/causalNLP/cladder","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"claw-eval":{"id":"claw-eval","name":"Claw-Eval","topic":"Agentic assistant tasks","description":"Benchmark of end-to-end OpenClaw-style personal-agent tasks; overall (EN) score.","url":"https://github.com/claw-eval/claw-eval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"claw-eval-multimodal":{"id":"claw-eval-multimodal","name":"Claw-Eval Multimodal","topic":"Multimodal agent tasks","description":"Multimodal subset of Claw-Eval OpenClaw-style agent benchmark.","url":"https://github.com/claw-eval/claw-eval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","Multimodal","Vision"]},"claw-gym":{"id":"claw-gym","name":"Claw-Gym","topic":"Agentic assistant tasks","description":"OpenClaw-style personal-assistant agent environment suite measuring task success on daily tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"claweval-pass-at-3":{"id":"claweval-pass-at-3","name":"ClawEval (Pass@3)","topic":"Agent tasks (OpenClaw)","description":"ClawEval OpenClaw agent benchmark, pass@3 (any of 3 trials succeeds).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"claweval-pass3":{"id":"claweval-pass3","name":"ClawEval (Pass^3)","topic":"Agent tasks (OpenClaw)","description":"ClawEval OpenClaw agent benchmark, pass^3 (all 3 trials succeed).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"click":{"id":"click","name":"CLIcK","topic":"Korean instruction following","description":"Korean long-form instruction-following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Korean","Instruction Following","Text"]},"clinc150-oos":{"id":"clinc150-oos","name":"CLINC150+OOS","topic":"Intent classification","description":"CLINC150 intent detection with out-of-scope class; macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/clinc/oos-eval","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Classification","Text"]},"cloc-count":{"id":"cloc-count","name":"CLOC Counting","topic":"Visual counting","description":"CLOC test object counting, exact-match accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Counting"]},"clockbench":{"id":"clockbench","name":"ClockBench","topic":"Multimodal reasoning","description":"ClockBench evaluates whether models can read analog clocks - a task that is trivial for humans, but current frontier models struggle with.","url":"https://clockbench.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"cloningscenarios":{"id":"cloningscenarios","name":"CloningScenarios","topic":"Biosecurity refusal","description":"Safety benchmark that red-teams models with cloning-related misuse scenarios to measure compliance and refusal rates.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Biosecurity","Safety","Text"]},"cluewsc":{"id":"cluewsc","name":"CLUEWSC","topic":"Coreference reasoning (Chinese)","description":"Chinese Winograd Schema-style coreference benchmark from CLUE.","url":"https://www.cluebenchmarks.com/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Reasoning","Text"]},"cmath":{"id":"cmath","name":"CMath","topic":"Math (Chinese)","description":"Chinese mathematics benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Math","STEM","Text"]},"cmmlu":{"id":"cmmlu","name":"CMMLU","topic":"Chinese multi-domain","description":"Chinese counterpart to MMLU.","url":"https://github.com/haonan-li/CMMLU","stars":"781","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Knowledge","QA","Text"]},"cmmmu":{"id":"cmmmu","name":"CMMMU","topic":"Chinese multimodal understanding","description":"Chinese Massive Multi-discipline Multimodal Understanding benchmark.","url":"https://cmmmu-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2401.11944","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Multimodal","Vision"]},"cmrc2018":{"id":"cmrc2018","name":"CMRC 2018","topic":"Chinese reading comprehension","description":"Span-extraction Chinese machine reading comprehension (CMRC 2018).","url":"https://github.com/ymcui/cmrc2018","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1810.07366","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Reading comprehension","Text"]},"cmt-benchmark":{"id":"cmt-benchmark","name":"CMT-Benchmark","topic":"Physics reasoning (condensed matter)","description":"Condensed Matter Theory benchmark of expert-written research-level theoretical physics problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Science","Text"]},"cnmo-2024":{"id":"cnmo-2024","name":"CNMO 2024","topic":"Math (competition)","description":"China National Mathematical Olympiad 2024 evaluation set.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"coco-caption":{"id":"coco-caption","name":"COCO Captions","topic":"Image captioning","description":"COCO image captioning benchmark (Karpathy test split), CIDEr score.","url":"https://cocodataset.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"cider","higher_is_better":true,"classification":["Vision","Captioning"]},"coco-det-f1":{"id":"coco-det-f1","name":"COCO Detection (F1)","topic":"Object detection","description":"COCO object detection with an MLLM; F1 averaged over IoU 0.5:0.95.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"coco-pointing":{"id":"coco-pointing","name":"COCO Pointing","topic":"Object pointing","description":"COCO object pointing accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"codeforces":{"id":"codeforces","name":"Codeforces","topic":"Competitive programming","description":"Competitive programming performance on Codeforces problems (ELO).","url":"https://codeforces.com/","stars":"","updated_at":"","leaderboard":"https://llmdb.com/benchmarks/codeforces","github":"","paper":"","modality":"text","metric":"ELO","higher_is_better":true,"classification":["Code","Text"]},"codeforces-percentile":{"id":"codeforces-percentile","name":"Codeforces (Percentile)","topic":"Competitive programming","description":"Competitive programming performance on Codeforces problems, reported as the percentile of human competitors outperformed.","url":"https://codeforces.com/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"codeif-bench":{"id":"codeif-bench","name":"CodeIF-Bench","topic":"Code instruction following","description":"Code-focused instruction following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Instruction Following","Text"]},"collegemath":{"id":"collegemath","name":"CollegeMath","topic":"College-level math","description":"College-level mathematics problems (MathScale) covering algebra, calculus, linear algebra, probability, etc.","url":"https://github.com/microsoft/unilm/tree/master/mathscale","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2403.02884","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Text"]},"collie":{"id":"collie","name":"COLLIE","topic":"Instruction following","description":"Comprehensive instruction-following evaluation suite.","url":"https://collie-benchmark.github.io/","stars":"55","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"collie-hard":{"id":"collie-hard","name":"Collie-Hard","topic":"Instruction following","description":"Hard subset of Collie instruction-following tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"commerceagentbench":{"id":"commerceagentbench","name":"CommerceAgentBench","topic":"E-commerce agents","description":"Agentic benchmark of B2B/e-commerce sourcing and commerce workflows (Accio-Lab).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"commonsenseqa":{"id":"commonsenseqa","name":"CommonsenseQA","topic":"Commonsense QA","description":"Multiple-choice QA requiring commonsense knowledge.","url":"https://www.tau-nlp.org/csqa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"complex-workflow":{"id":"complex-workflow","name":"Complex Workflow","topic":"Complex workflows","description":"Complex workflow benchmark for economically valuable tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Text"]},"complexfuncbench":{"id":"complexfuncbench","name":"ComplexFuncBench","topic":"Code reasoning","description":"Complex function reasoning and synthesis benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"complexfuncbench-audio":{"id":"complexfuncbench-audio","name":"ComplexFuncBench (Audio)","topic":"Spoken function calling","description":"ComplexFuncBench multi-step, constrained function calling with spoken user requests (%).","url":"https://github.com/THUDM/ComplexFuncBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Tool calling"]},"cona":{"id":"cona","name":"CoNA","topic":"Safety / refusal","description":"Compliance-on-Non-Assist (CoNA): evaluates refusal in disallowed content scenarios.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2309.07875","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Refusal","Text"]},"contphy":{"id":"contphy","name":"ContPhy","topic":"Video physical reasoning","description":"Continuum Physical Concept Learning and Reasoning from videos.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Video","Reasoning"]},"contract-scrub":{"id":"contract-scrub","name":"Contract Scrub","topic":"Contract review","description":"Contract-review task: identify and redact/flag clauses in legal contracts (Thomson Reuters eval).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Text"]},"contractnli":{"id":"contractnli","name":"ContractNLI","topic":"Legal NLI","description":"Document-level natural language inference over contracts; macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://stanfordnlp.github.io/contract-nli/","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["NLI","Legal","Text"]},"convfinqa":{"id":"convfinqa","name":"ConvFinQA","topic":"Conversational QA","description":"Conversational numerical reasoning over financial reports (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"copa":{"id":"copa","name":"COPA","topic":"Causal reasoning","description":"Choice of Plausible Alternatives.","url":"https://people.ict.usc.edu/~gordon/copa.html","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"coqa":{"id":"coqa","name":"CoQA","topic":"Conversational QA","description":"Conversational question answering (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"core":{"id":"core","name":"CORE","topic":"Ontological reasoning","description":"Comprehensive Ontological Relation Evaluation for Large Language Models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Knowledge","Text"]},"corpfin":{"id":"corpfin","name":"CorpFin","topic":"Corporate finance","description":"Corporate finance reasoning and numerical literacy benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Reasoning","Text"]},"corpusqa":{"id":"corpusqa","name":"CorpusQA","topic":"Long-context QA","description":"Question answering over large text corpora.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","QA","Text"]},"cosmos-reason-bench":{"id":"cosmos-reason-bench","name":"COSMOS (Cosmos-Reason benchmark)","topic":"Embodied reasoning","description":"NVIDIA Cosmos-Reason embodied physical-reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"countbench":{"id":"countbench","name":"CountBench","topic":"Visual counting","description":"Object counting and numeracy benchmark for visual-language models across varied scenes.","url":"https://huggingface.co/datasets/nielsr/countbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2302.12066","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"countbenchqa":{"id":"countbenchqa","name":"CountBenchQA","topic":"Visual counting QA","description":"Visual question answering benchmark focused on counting objects across varied scenes.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Counting","Multimodal","QA","Vision"]},"countdown":{"id":"countdown","name":"Countdown","topic":"Planning and reasoning","description":"Countdown-style reasoning and planning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"countix":{"id":"countix","name":"Countix","topic":"Video counting","description":"Video-based counting benchmark for multiple objects.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Counting","Video","Vision"]},"countqa":{"id":"countqa","name":"CountQA","topic":"Visual counting","description":"Visual question answering benchmark on object counting in real-world images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal"]},"covost":{"id":"covost","name":"CoVoST","topic":"Speech translation","description":"CoVoST speech-to-text translation benchmark (BLEU).","url":"https://github.com/facebookresearch/covost","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"bleu","higher_is_better":true,"classification":["Audio","Translation"]},"crag":{"id":"crag","name":"CRAG","topic":"Retrieval QA","description":"Complex Retrieval-Augmented Generation benchmark for grounded question answering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Retrieval","QA","Text"]},"creative-story-writing-v3":{"id":"creative-story-writing-v3","name":"Creative Story‑Writing Benchmark V3","topic":"Creative writing","description":"Story writing benchmark evaluating creativity, coherence, and style (v3).","url":"","stars":"291","updated_at":"","leaderboard":"https://github.com/lechmazur/writing","github":"https://github.com/lechmazur/writing","paper":"","modality":"text","metric":"score","higher_is_better":true,"classification":["Text"]},"creative-writing-longform":{"id":"creative-writing-longform","name":"Longform Creative Writing","topic":"Creative writing","description":"Longform creative writing evaluation (EQ-Bench).","url":"https://eqbench.com/creative_writing_longform.html","stars":"20","updated_at":"","leaderboard":"https://eqbench.com/creative_writing_longform.html","github":"https://github.com/EQ-bench","paper":"https://arxiv.org/abs/2312.06281","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"creative-writing-v3":{"id":"creative-writing-v3","name":"Creative Writing v3","topic":"Creative writing","description":"A LLM-judged creative writing benchmark.","url":"https://eqbench.com/creative_writing.html","stars":"54","updated_at":"","leaderboard":"https://eqbench.com/creative_writing.html","github":"https://github.com/EQ-bench","paper":"https://arxiv.org/abs/2312.06281","modality":"text","metric":"ELO","higher_is_better":true,"classification":["Text"]},"critpt":{"id":"critpt","name":"Complex Research using Integrated Thinking – Physics Test","topic":"Reasoning","description":"CritPt (Complex Research using Integrated Thinking – Physics Test) benchmark.","url":"https://critpt.com/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"crosstabbench-relation-f1":{"id":"crosstabbench-relation-f1","name":"CrossTabBench Relation F1","topic":"Cross-page tables","description":"CrossTabBench: F1 for detecting cross-page table continuation relations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"crosstabbench-teds":{"id":"crosstabbench-teds","name":"CrossTabBench Reconstruction TEDS","topic":"Cross-page tables","description":"CrossTabBench: TEDS of reconstructed cross-page tables.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"crossvid":{"id":"crossvid","name":"CrossVid","topic":"Multi-video reasoning","description":"Benchmark for cross-video (multi-video) understanding and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Video","Reasoning"]},"crpe":{"id":"crpe","name":"CRPE","topic":"Relation hallucination","description":"Circular-based Relation Probing Evaluation (relation split) for object-relation hallucination.","url":"https://github.com/OpenGVLab/all-seeing","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2402.19474","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Hallucination","Multimodal","Vision"]},"crux-i":{"id":"crux-i","name":"CRUX-I","topic":"Code reasoning","description":"Code Reasoning and Understanding eXam – Interactive.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"https://crux-eval.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Exams","Reasoning","Text"]},"crux-o":{"id":"crux-o","name":"CRUX-O","topic":"Code reasoning","description":"Code Reasoning and Understanding eXam – Offline.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"https://crux-eval.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Exams","Reasoning","Text"]},"cruxeval":{"id":"cruxeval","name":"CruxEval","topic":"Code reasoning","description":"Mathematical coding challenge set from the CruxEval benchmark.","url":"https://github.com/crux-eval/cruxeval","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/crux-eval/cruxeval","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"cruxeval-decision":{"id":"cruxeval-decision","name":"CRUXEval (decision)","topic":"Code reasoning","description":"CRUXEval output prediction posed as choices among candidate outputs; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/crux-eval/cruxeval","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Decision","Text"]},"csatqa":{"id":"csatqa","name":"CSATQA","topic":"Korean exams","description":"Questions from the Korean College Scholastic Ability Test (reading, grammar, writing).","url":"https://huggingface.co/datasets/HAERAE-HUB/csatqa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Korean","Text"]},"csimpleqa":{"id":"csimpleqa","name":"CSimpleQA","topic":"QA","description":"Chinese SimpleQA benchmark variant (short factual questions).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Knowledge","QA","Text"]},"cultcat":{"id":"cultcat","name":"CultCat","topic":"Cultural knowledge","description":"Yandex 4-shot benchmark of (Russian) cultural facts.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"cultural-bench":{"id":"cultural-bench","name":"Cultural Bench","topic":"Cultural understanding","description":"Evaluation of cultural knowledge and norms across regions and topics.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"cursorbench-3.0":{"id":"cursorbench-3.0","name":"CursorBench 3.0","topic":"Software engineering","description":"Cursor's coding-agent benchmark on ambiguous multi-file tasks from real Cursor sessions (v3.0, used in the Composer 2 technical report as CursorBench-3).","url":"https://cursor.com/cursorbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2603.24477","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"cursorbench-3.2":{"id":"cursorbench-3.2","name":"CursorBench 3.2","topic":"Software engineering","description":"Cursor's coding-agent benchmark on ambiguous multi-file tasks from real Cursor sessions (v3.2).","url":"https://cursor.com/cursorbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"cursorbench-4.0":{"id":"cursorbench-4.0","name":"CursorBench 4.0","topic":"Software engineering","description":"Cursor's coding-agent benchmark on ambiguous multi-file tasks from real Cursor sessions (v4.0).","url":"https://cursor.com/cursorbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"customer-support-qa":{"id":"customer-support-qa","name":"Customer Support Q&A","topic":"Customer support QA","description":"Customer support question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","QA","Text"]},"cute":{"id":"cute","name":"CUTE","topic":"English characters","description":"CUTE aggregate capability score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text","English"]},"cv-bench":{"id":"cv-bench","name":"CV-Bench","topic":"Computer vision QA","description":"Diverse CV tasks for VLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"cvtg-2k-clipscore":{"id":"cvtg-2k-clipscore","name":"CVTG-2K CLIPScore","topic":"Text rendering","description":"CVTG-2K CLIPScore for text rendering in image generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"cvtg-2k-ned":{"id":"cvtg-2k-ned","name":"CVTG-2K NED","topic":"Text rendering","description":"CVTG-2K normalized edit distance (NED) for text rendering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"cvtg-2k-word-accuracy":{"id":"cvtg-2k-word-accuracy","name":"CVTG-2K Word Accuracy","topic":"Text rendering","description":"CVTG-2K word accuracy for text rendering in images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"cwe-bench":{"id":"cwe-bench","name":"CWE-Bench","topic":"Cybersecurity","description":"CWE-Bench v1 vulnerability detection benchmark (cwe-bench.com, Collinear AI); pass@1.","url":"https://cwe-bench.com/","stars":"","updated_at":"","leaderboard":"https://cwe-bench.com/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Security","Text"]},"cxr14":{"id":"cxr14","name":"ChestX-ray14","topic":"Medical imaging","description":"NIH ChestX-ray14 classification, macro F1.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"cybench":{"id":"cybench","name":"CyBench","topic":"Cybersecurity CTF","description":"Framework with 40 professional-level CTF tasks evaluating LLMs' practical cybersecurity capabilities.","url":"https://cybench.github.io/","stars":"","updated_at":"","leaderboard":"https://cybench.github.io/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Text"]},"cyberattack-assistance":{"id":"cyberattack-assistance","name":"CyberattackAssistance","topic":"Cybersecurity","description":"Assesses a model's tendency to assist with cyberattacks or unsafe activity.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Security","Safety","Text"]},"cybergym":{"id":"cybergym","name":"CyberGym","topic":"Cybersecurity tasks","description":"Benchmark for cybersecurity-related coding and reasoning tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Security","Text"]},"cybersecurity-capture-the-flag-challenges":{"id":"cybersecurity-capture-the-flag-challenges","name":"Cybersecurity Capture The Flag Challenges","topic":"Cybersecurity CTF","description":"Capture-the-flag challenge benchmark evaluating cybersecurity problem-solving skills.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"cybersecurity-ctf":{"id":"cybersecurity-ctf","name":"Cybersecurity CTF","topic":"Cybersecurity CTF","description":"Cybersecurity Capture The Flag challenges benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Security","Text"]},"d4la":{"id":"d4la","name":"D4LA","topic":"Layout analysis","description":"Diverse and detailed dataset for document layout analysis (12 document types, 27 categories).","url":"https://arxiv.org/abs/2308.14978","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Vision","Document"]},"da-2k":{"id":"da-2k","name":"DA-2K","topic":"Spatial reasoning","description":"2D/3D spatial reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"daily-omni":{"id":"daily-omni","name":"Daily-Omni","topic":"Audio-visual QA","description":"Daily-Omni: audio-visual question answering on everyday videos with temporal alignment.","url":"https://arxiv.org/abs/2505.17862","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Audio","QA"]},"deep-planning":{"id":"deep-planning","name":"Deep Planning","topic":"Planning and reasoning","description":"Benchmark evaluating deep planning and multi-step reasoning capabilities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Planning","Reasoning","Text"]},"deepcodebench":{"id":"deepcodebench","name":"DeepCodeBench","topic":"Code QA","description":"Qodo's question-answering benchmark over real-world code repositories, requiring retrieval and reasoning across large codebases.","url":"https://huggingface.co/datasets/Qodo/deep_code_bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"deepconsult":{"id":"deepconsult","name":"DeepConsult","topic":"Agentic writing","description":"Agentic consulting and writing benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Writing","Agents","Text"]},"deepform":{"id":"deepform","name":"DeepForm","topic":"Document key information extraction","description":"DeepForm key information extraction from political ad disclosure forms (F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2303.13839","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Vision"]},"deepmind-math":{"id":"deepmind-math","name":"DeepMind Mathematics","topic":"Math reasoning","description":"Synthetic math problem sets from DeepMind covering arithmetic, algebra, calculus, and more.","url":"https://github.com/deepmind/mathematics_dataset","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/deepmind/mathematics_dataset","paper":"https://arxiv.org/abs/1904.01557","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"deepresearch-bench-ii":{"id":"deepresearch-bench-ii","name":"DeepResearch Bench II","topic":"Deep research reports","description":"Second-generation DeepResearch Bench evaluating long-form deep-research reports with fine-grained rubrics.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Research","Text"]},"deepresearchbench":{"id":"deepresearchbench","name":"DeepResearchBench","topic":"Agentic research writing","description":"Research-oriented agentic writing and planning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Writing","Agents","Text"]},"deepsearchqa":{"id":"deepsearchqa","name":"DeepSearchQA","topic":"Deep web search QA","description":"Multi-step web search and question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Search","QA","Text"]},"deepsearchqa-f1":{"id":"deepsearchqa-f1","name":"DeepSearchQA (F1)","topic":"Deep web search QA","description":"DeepSearchQA multi-step web search benchmark, F1-score metric (the existing deepsearchqa entry is used for the accuracy metric).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Search","Web","Text"]},"deepshop":{"id":"deepshop","name":"DeepShop","topic":"Web agents","description":"Online shopping web-agent benchmark with complex, deep product-search queries.","url":"https://arxiv.org/abs/2506.02839","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2506.02839","modality":"visual","metric":"percent","higher_is_better":true,"classification":""},"deepswe":{"id":"deepswe","name":"DeepSWE","topic":"Software engineering","description":"113 long-horizon software engineering tasks written from scratch to avoid contamination (v1.1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"deepswe-v1.0":{"id":"deepswe-v1.0","name":"DeepSWE 1.0","topic":"Software engineering","description":"Datacurve DeepSWE v1.0 long-horizon software-engineering tasks, run by Artificial Analysis with each provider's own harness (pass@1).","url":"https://deepswe.datacurve.ai/","stars":"","updated_at":"","leaderboard":"https://deepswe.datacurve.ai/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"deepswe-v1.1":{"id":"deepswe-v1.1","name":"DeepSWE v1.1","topic":"Software engineering","description":"Original long-horizon software-engineering tasks in real codebases (Datacurve).","url":"https://deepswe.datacurve.ai/","stars":"","updated_at":"","leaderboard":"https://deepswe.datacurve.ai/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"defects4j":{"id":"defects4j","name":"Defects4J","topic":"Program repair","description":"Real Java bugs from Defects4J used for automated program repair.","url":"https://github.com/rjust/defects4j","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"dense200-det-f1":{"id":"dense200-det-f1","name":"Dense200 Detection (F1)","topic":"Object detection","description":"Dense200 object detection with an MLLM; F1 averaged over IoU 0.5:0.95.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"dense200-pointing":{"id":"dense200-pointing","name":"Dense200 Pointing","topic":"Object pointing","description":"Dense200 object pointing accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"der2-bench":{"id":"der2-bench","name":"DeR2 Bench","topic":"Long-context reasoning","description":"Dense retrieval and reasoning benchmark for long-context evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"design2code":{"id":"design2code","name":"Design2Code","topic":"Coding (UI)","description":"Translating UI designs into code.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","UI","Text"]},"designarena":{"id":"designarena","name":"DesignArena","topic":"Generative design","description":"Leaderboard tracking generative design systems across layout, branding, and marketing tasks.","url":"https://designarena.ai","stars":"","updated_at":"","leaderboard":"https://designarena.ai","github":"","paper":"","modality":"visual","metric":"elo","higher_is_better":true,"classification":["Design","Vision"]},"detailbench":{"id":"detailbench","name":"DetailBench","topic":"Spot small mistakes","description":"Evaluates whether LLMs can notice subtle errors and minor inconsistencies in text.","url":"https://huggingface.co/datasets/xeophon/detailbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"dialogue-safety":{"id":"dialogue-safety","name":"DialogueSafety","topic":"Safety / dialogue","description":"Dialogue safety evaluation across conversational prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Dialogue","Text"]},"dices350":{"id":"dices350","name":"DICES350","topic":"Safety / robustness","description":"Conversational robustness across dialogue safety datasets (DICES350).","url":"","stars":"26","updated_at":"","leaderboard":"","github":"https://github.com/google-research-datasets/dices-dataset","paper":"https://arxiv.org/abs/2306.11247","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Dialogue","Robustness","Text"]},"discox":{"id":"discox","name":"DiscoX","topic":"Agentic writing","description":"DiscoX benchmark for agentic writing and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Writing","Agents","Text"]},"do-anything-now":{"id":"do-anything-now","name":"Do-Anything-Now","topic":"Safety / jailbreak","description":"Resistance to Do Anything Now (DAN) style jailbreak prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"do-not-answer":{"id":"do-not-answer","name":"Do-Not-Answer","topic":"Safety / refusal","description":"Evaluates a model's ability to refuse unsafe or disallowed requests.","url":"https://huggingface.co/datasets/LibrAI/do-not-answer","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/Libr-AI/do-not-answer","paper":"https://aclanthology.org/2024.findings-eacl.61","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Refusal","Text"]},"doc2dial":{"id":"doc2dial","name":"Doc2Dial","topic":"Conversational QA","description":"Document-grounded dialogue QA (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"docfigcapbench-caption":{"id":"docfigcapbench-caption","name":"DocFigCapBench Caption Score","topic":"Document figure captioning","description":"DocFigCapBench: caption quality score (0-10 scale).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["OCR","Document","Vision"]},"docfigcapbench-loc-f1":{"id":"docfigcapbench-loc-f1","name":"DocFigCapBench Localization F1","topic":"Document figure captioning","description":"DocFigCapBench: localization F1 of document figures.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"doclaynet":{"id":"doclaynet","name":"DocLayNet","topic":"Layout analysis","description":"Human-annotated document layout segmentation dataset; layout analysis mIoU/mAP.","url":"https://github.com/DS4SD/DocLayNet","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Vision","Document"]},"docmath":{"id":"docmath","name":"DocMath","topic":"Document math","description":"Math reasoning on document-based problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Math","Multimodal","Vision"]},"docmatix":{"id":"docmatix","name":"Docmatix (multimodal RAG)","topic":"Multimodal RAG","description":"Docmatix document-VQA based multimodal retrieval-augmented QA evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["RAG","Vision"]},"docvqa":{"id":"docvqa","name":"DocVQA","topic":"Document understanding (VQA)","description":"Visual question answering over scanned documents.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Knowledge","Multimodal","QA","Vision"]},"dolphin-page":{"id":"dolphin-page","name":"Dolphin-Page","topic":"Document OCR","description":"Dolphin Page benchmark measuring OCR fidelity and structured extraction on multi-layout documents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Document","OCR","Vision"]},"doqa":{"id":"doqa","name":"DoQA","topic":"Conversational QA","description":"Domain-specific FAQ conversational QA (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"dpg-bench":{"id":"dpg-bench","name":"DPG-Bench","topic":"Text rendering","description":"DPG-Bench score for text rendering in image generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"draco":{"id":"draco","name":"DRACO","topic":"Deep research","description":"DRACO deep-research benchmark scored with per-question rubrics.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Research","Text"]},"dream-1k":{"id":"dream-1k","name":"DREAM-1K","topic":"Video captioning","description":"DREAM-1K dense/detailed video description benchmark of 1,000 clips.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"drop":{"id":"drop","name":"DROP","topic":"Reading + reasoning","description":"Discrete reasoning over paragraphs (addition, counting, comparisons).","url":"https://allenai.org/data/drop","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"ds-1000":{"id":"ds-1000","name":"DS-1000","topic":"Data-science code generation","description":"DS-1000: 1000 data-science coding problems across 7 Python libraries.","url":"https://ds1000-code-gen.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2211.11501","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"dsbench-fullstack":{"id":"dsbench-fullstack","name":"DSBench-FullStack (internal)","topic":"Coding","description":"DeepSeek internal full-stack development test set for coding agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Coding","Agents"]},"dsbench-hard":{"id":"dsbench-hard","name":"DSBench-Hard (internal)","topic":"Coding","description":"DeepSeek internal test set of difficult coding-agent problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Coding","Agents"]},"dude":{"id":"dude","name":"DUDE","topic":"Multimodal long-context","description":"Long-context multimodal understanding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Long Context","Multimodal","Vision"]},"dynamath":{"id":"dynamath","name":"DynaMath","topic":"Math reasoning (video)","description":"Dynamic/video-based mathematical reasoning evaluating temporal and visual understanding.","url":"https://dynamath.github.io/","stars":"","updated_at":"","leaderboard":"https://dynamath.github.io/","github":"https://github.com/DynaMath/DynaMath","paper":"https://arxiv.org/pdf/2411.00836","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Reasoning","STEM","Video","Vision"]},"eclektic":{"id":"eclektic","name":"ECLeKTic","topic":"Cross-lingual knowledge","description":"Cross-lingual knowledge transfer benchmark across languages.","url":"https://arxiv.org/abs/2502.21228","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"economically-important-tasks":{"id":"economically-important-tasks","name":"Economically important tasks","topic":"Industry QA (cross-domain)","description":"Evaluation suite of real-world, economically impactful tasks across key industries and workflows.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Industry","Knowledge","QA","Text"]},"edubench-english":{"id":"edubench-english","name":"EduBench English","topic":"Education (English)","description":"Yandex education benchmark built from Alice user queries: English language subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"edubench-history":{"id":"edubench-history","name":"EduBench History","topic":"Education (history)","description":"Yandex education benchmark built from Alice user queries: history subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"edubench-literature":{"id":"edubench-literature","name":"EduBench Literature","topic":"Education (literature)","description":"Yandex education benchmark built from Alice user queries: literature subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"edubench-math":{"id":"edubench-math","name":"EduBench Math","topic":"Education (math)","description":"Yandex education benchmark built from Alice user queries: school math subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"edubench-math-university":{"id":"edubench-math-university","name":"EduBench Math University","topic":"Education (university math)","description":"Yandex education benchmark built from Alice user queries: university math subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"edubench-russian":{"id":"edubench-russian","name":"EduBench Russian","topic":"Education (Russian language)","description":"Yandex education benchmark built from Alice user queries: Russian language subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"education-bench":{"id":"education-bench","name":"Education","topic":"Economics/education","description":"Education field evaluation (economically valuable tasks).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Text"]},"eebench":{"id":"eebench","name":"EEBench","topic":"Electrical engineering","description":"atopile's electrical-engineering agent benchmark: circuit/PCB design tasks expressed as declarative atopile code and verified automatically.","url":"https://eebench.org/","stars":"","updated_at":"","leaderboard":"https://eebench.org/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Engineering","Agents","Text"]},"ege":{"id":"ege","name":"EGE","topic":"Russian state exam","description":"Part-A questions of the Russian Unified State Exam (EGE) across subjects.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"ego3d-bench":{"id":"ego3d-bench","name":"Ego3D-Bench (RMSE)","topic":"Driving spatial reasoning","description":"Ego-centric multi-view 3D spatial reasoning benchmark on outdoor driving scenes; metric = RMSE of numeric answers (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"rmse","higher_is_better":false,"classification":["Autonomous Driving","Spatial","Vision"]},"egoplan-bench2":{"id":"egoplan-bench2","name":"EgoPlan-Bench2","topic":"Embodied reasoning","description":"Egocentric embodied task-planning benchmark (v2).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"egoschema":{"id":"egoschema","name":"EgoSchema","topic":"Egocentric video QA","description":"EgoSchema validation accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"egotempo":{"id":"egotempo","name":"EgoTempo","topic":"Egocentric temporal reasoning","description":"Egocentric video temporal reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Temporal","Vision"]},"ehrnoteqa":{"id":"ehrnoteqa","name":"EHRNoteQA","topic":"Medical records","description":"Clinical-note question answering over MIMIC-IV discharge summaries.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"eifbench":{"id":"eifbench","name":"EIFBench","topic":"Instruction following","description":"Complex instruction-following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"emb-v1.1":{"id":"emb-v1.1","name":"EMB v1.1","topic":"Financial modeling agents","description":"Vals AI Excel Modeling Benchmark v1.1: agents build Excel-based financial models.","url":"https://www.vals.ai/benchmarks/emb-v1-1","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"embspatialbench":{"id":"embspatialbench","name":"EmbSpatialBench","topic":"Spatial understanding","description":"Embodied spatial understanding benchmark evaluating navigation and localization.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Embodied","Vision"]},"emma":{"id":"emma","name":"EMMA","topic":"Multimodal reasoning","description":"EMMA benchmark for multimodal reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"enamel":{"id":"enamel","name":"Enamel","topic":"Composite capability","description":"Composite capability benchmark capturing broad model performance (Enamel score).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Composite","Reasoning","Text"]},"enconda-bench":{"id":"enconda-bench","name":"EnConda-Bench","topic":"Code editing","description":"English code editing benchmark for applying conditional modifications.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Editing","Text"]},"encyclo-k":{"id":"encyclo-k","name":"Encyclo-K","topic":"Encyclopedic knowledge","description":"Encyclopedic knowledge evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"enigmaval":{"id":"enigmaval","name":"EnigmaEval","topic":"Challenging puzzles","description":"Challenging puzzle benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Puzzles","Reasoning","Vision"]},"enterprise-rag":{"id":"enterprise-rag","name":"Enterprise RAG","topic":"Retrieval-augmented generation","description":"Enterprise retrieval-augmented generation evaluation covering internal knowledge bases.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["RAG","Knowledge","Text"]},"enterpriseops-gym":{"id":"enterpriseops-gym","name":"EnterpriseOps-Gym","topic":"Enterprise agent tasks","description":"Agentic benchmark of enterprise operations workflows with tool use.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"eqbench":{"id":"eqbench","name":"EQ-Bench","topic":"Reasoning","description":"General reasoning benchmark assessing equation/logic capabilities.","url":"https://eqbench.com/eqbench-v2.html","stars":"352","updated_at":"","leaderboard":"https://eqbench.com/eqbench-v2.html","github":"https://github.com/EQ-bench","paper":"https://arxiv.org/abs/2312.06281","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"eqbench-3":{"id":"eqbench-3","name":"EQ-Bench 3","topic":"Emotional intelligence (roleplay)","description":"A benchmark measuring emotional intelligence in challenging roleplays, judged by Sonnet 3.7.","url":"https://eqbench.com/index.html","stars":"21","updated_at":"","leaderboard":"https://eqbench.com/index.html","github":"https://github.com/EQ-bench","paper":"https://arxiv.org/abs/2312.06281","modality":"text","metric":"ELO","higher_is_better":true,"classification":["Reasoning","Text"]},"erqa":{"id":"erqa","name":"ERQA","topic":"Spatial reasoning","description":"Spatial recognition and reasoning QA benchmark (ERQA).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Reasoning","Vision"]},"erqa-plus":{"id":"erqa-plus","name":"ERQA-PLUS","topic":"Embodied reasoning","description":"Extended Embodied Reasoning QA benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"et-bench":{"id":"et-bench","name":"E.T. Bench","topic":"Video event understanding","description":"E.T. Bench: event-level and time-sensitive video understanding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"evalperf":{"id":"evalperf","name":"EvalPerf","topic":"Code evaluation performance","description":"Measures performance of LLM code evaluation, including runtime, memory, and efficiency metrics.","url":"https://evalplus.github.io/evalperf.html","stars":"","updated_at":"","leaderboard":"https://evalplus.github.io/evalperf.html","github":"https://github.com/evalplus/evalplus","paper":"https://openreview.net/forum?id=IBCBMeAhmC","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"evalplus":{"id":"evalplus","name":"EvalPlus","topic":"Code generation","description":"Aggregated code evaluation suite from EvalPlus.","url":"https://github.com/evalplus/evalplus","stars":"1577","updated_at":"","leaderboard":"https://evalplus.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"evg":{"id":"evg","name":"EVG","topic":"Document OCR","description":"EVG document OCR benchmark evaluating recognition accuracy and layout extraction.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Document","OCR","Vision"]},"execute":{"id":"execute","name":"EXECUTE","topic":"Multilingual character tasks","description":"Multilingual character-level evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"expertfactsqa-law":{"id":"expertfactsqa-law","name":"ExpertFactsQA Law","topic":"Expert knowledge (law)","description":"Yandex expert-written factual QA benchmark, law (hard) subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"expertfactsqa-medicine":{"id":"expertfactsqa-medicine","name":"ExpertFactsQA Medicine","topic":"Expert knowledge (medicine)","description":"Yandex expert-written factual QA benchmark, medicine subset.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"exploitbench":{"id":"exploitbench","name":"ExploitBench","topic":"Cybersecurity","description":"Developing V8 vulnerabilities into increasingly capable exploits (capability coverage).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"exploitbench-2026-jun-aug":{"id":"exploitbench-2026-jun-aug","name":"ExploitBench (Jun-Aug 2026)","topic":"Cybersecurity","description":"ExploitBench restricted to vulnerabilities disclosed June-August 2026, after model training cutoffs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"exploitgym":{"id":"exploitgym","name":"ExploitGym","topic":"Cybersecurity","description":"UC Berkeley-led benchmark: agents turn real known vulnerabilities into working exploits achieving arbitrary code execution.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"exploitgym-honeypot":{"id":"exploitgym-honeypot","name":"ExploitGym-Honeypot","topic":"Safety","description":"ExploitGym honeypot variant measuring how often agents take bait out of scope (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Security","Text"]},"facts-benchmark-suite":{"id":"facts-benchmark-suite","name":"FACTS Benchmark Suite","topic":"Held out internal grounding, parametric, MM, and search retrieval benchmarks","description":"Comprehensive factuality benchmark suite covering held-out internal grounding, parametric knowledge, multimodal understanding, and search retrieval benchmarks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Grounding","Knowledge","QA","Text"]},"facts-grounding":{"id":"facts-grounding","name":"FACTS Grounding","topic":"Grounding / factuality","description":"Grounded factuality benchmark evaluating model alignment with source facts.","url":"https://goo.gle/FACTS_paper","stars":"","updated_at":"2025-09-08","leaderboard":"https://www.kaggle.com/benchmarks/google/facts-grounding","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Grounding","Instruction Following","Text"]},"facts-parametric":{"id":"facts-parametric","name":"FACTS Parametric","topic":"Factuality","description":"FACTS suite parametric-knowledge split: closed-book factoid questions answered without retrieval.","url":"https://www.kaggle.com/benchmarks/google/facts","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Factuality"]},"factscore":{"id":"factscore","name":"FActScore","topic":"Factuality on open-source prompts","description":"Measures factuality on an open-source prompt suite; scored as 100 - hallucination rate, higher is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Safety","Text"]},"fairpoliticsqa":{"id":"fairpoliticsqa","name":"FairPoliticsQA","topic":"Political fairness (Japanese)","description":"Sakana AI benchmark of balanced, fair answers to Japanese political questions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Alignment","Text"]},"faitheval-inconsistent":{"id":"faitheval-inconsistent","name":"FaithEval (Inconsistent)","topic":"Context faithfulness","description":"FaithEval inconsistent-context subset: detecting contradictory evidence in retrieved context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Text"]},"faithjudge-1-hallu":{"id":"faithjudge-1-hallu","name":"FaithJudge (1-Hallu.)","topic":"Hallucination detection","description":"FaithJudge hallucination rate with 1-hallucination metric (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Safety","Text"]},"faix-agent":{"id":"faix-agent","name":"Meta Score Agent","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-08-05","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Agents"]},"faix-code":{"id":"faix-code","name":"Meta Score Code","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-09-27","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Code"]},"faix-math":{"id":"faix-math","name":"Meta Score Math","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-08-05","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Math"]},"faix-ocr":{"id":"faix-ocr","name":"Meta Score OCR","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-08-05","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Document"]},"faix-overall":{"id":"faix-overall","name":"Meta Score Overall","topic":"Composite capability index","description":"Overall Meta Score across the benchmarks of all Meta Score categories.","url":"","stars":"","updated_at":"2026-10-17","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate"]},"faix-safety":{"id":"faix-safety","name":"Meta Score Safety","topic":"Composite safety index","description":"","url":"","stars":"","updated_at":"2025-09-27","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Safety"]},"faix-stem":{"id":"faix-stem","name":"Meta Score STEM","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-08-05","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate"]},"faix-text":{"id":"faix-text","name":"Meta Score Text","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-08-05","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Text"]},"faix-visual":{"id":"faix-visual","name":"Meta Score Visual","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-08-05","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Vision"]},"faix-writing":{"id":"faix-writing","name":"Meta Score Writing","topic":"Composite capability index","description":"","url":"","stars":"","updated_at":"2025-09-27","leaderboard":"","github":"","paper":"","modality":"aggregate","metric":"percent","higher_is_better":true,"classification":["Composite","Aggregate","Writing"]},"fate-h":{"id":"fate-h","name":"FATE-H","topic":"Formal theorem proving (abstract algebra)","description":"FATE-H: graduate-level abstract algebra problems formalized in Lean 4 (100 problems).","url":"https://github.com/frenzymath/FATE","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Formal Proofs","Text"]},"fate-x":{"id":"fate-x","name":"FATE-X","topic":"Formal theorem proving (abstract algebra)","description":"FATE-X: PhD-level abstract algebra problems formalized in Lean 4 (100 problems).","url":"https://github.com/frenzymath/FATE","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Formal Proofs","Text"]},"fd-bench-v1-latency":{"id":"fd-bench-v1-latency","name":"Full-Duplex-Bench V1 Turn-taking Latency","topic":"Full-duplex speech interaction","description":"Full-Duplex-Bench v1 smooth turn-taking latency in seconds (lower is better).","url":"https://github.com/DanielLin94144/Full-Duplex-Bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"seconds","higher_is_better":false,"classification":["Audio","Voice","Latency"]},"fd-bench-v1.5":{"id":"fd-bench-v1.5","name":"FD-Bench V1.5","topic":"Full-duplex speech interaction","description":"Full-duplex spoken dialogue benchmark v1.5 (interruption handling, backchannel, turn-taking quality); average score.","url":"https://thinkingmachines.ai/blog/interaction-models/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Agents"]},"fd-bench-v3":{"id":"fd-bench-v3","name":"FD-Bench V3","topic":"Full-duplex speech + tool use","description":"Full-duplex spoken dialogue benchmark v3 with tool calling; response quality (%).","url":"https://thinkingmachines.ai/blog/interaction-models/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Agents"]},"fd-bench-v3-pass1":{"id":"fd-bench-v3-pass1","name":"FD-Bench V3 Pass@1","topic":"Full-duplex speech + tool use","description":"Full-Duplex-Bench v3: tool-calling task success (pass@1, %) in full-duplex spoken dialogue.","url":"https://github.com/DanielLin94144/Full-Duplex-Bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2604.04847","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Agents"]},"featbench":{"id":"featbench","name":"FeatBench","topic":"Code (feature implementation)","description":"Benchmark of feature implementation in real repositories from natural-language requirements (vibe coding).","url":"https://arxiv.org/abs/2509.22237","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2509.22237","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"fetaqa":{"id":"fetaqa","name":"FeTaQA","topic":"Table QA","description":"Free-form table QA (MMTab subset; BLEU).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"few-shot-attack":{"id":"few-shot-attack","name":"Few-Shot Attack","topic":"Safety / jailbreak","description":"Jailbreak resistance under few-shot adversarial prompting.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"figqa":{"id":"figqa","name":"FigQA","topic":"Figure understanding and QA","description":"Figure question answering benchmark evaluating visual reasoning over scientific figures and diagrams.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Science","Vision"]},"finance-reasoning":{"id":"finance-reasoning","name":"FinanceReasoning","topic":"Financial reasoning","description":"Financial reasoning benchmark evaluating quantitative and qualitative finance problem solving.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Reasoning","Text"]},"financeagent":{"id":"financeagent","name":"FinanceAgent","topic":"Agentic finance tasks","description":"Interactive financial agent benchmark requiring multi-step tool use.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"financeagent-v1.1":{"id":"financeagent-v1.1","name":"FinanceAgent v1.1","topic":"Agentic finance tasks","description":"Finance Agent v1.1 benchmark for interactive financial agent evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"financeagent-v2":{"id":"financeagent-v2","name":"FinanceAgent v2","topic":"Agentic finance tasks","description":"Vals AI Finance Agent Benchmark v2: agentic financial-research tasks over SEC filings; new dataset, not comparable to v1.1.","url":"https://www.vals.ai/benchmarks/finance_agent","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"financebench-fulldoc":{"id":"financebench-fulldoc","name":"FinanceBench (FullDoc)","topic":"Finance QA","description":"FinanceBench full-document question answering benchmark requiring long-context financial understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","QA","Long Context","Text"]},"financialphrasebank":{"id":"financialphrasebank","name":"FinancialPhraseBank","topic":"Financial sentiment","description":"Sentence-level financial news sentiment classification.","url":"https://huggingface.co/datasets/takala/financial_phrasebank","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Sentiment","Text"]},"finentity-decision":{"id":"finentity-decision","name":"FinEntity (decision)","topic":"Financial sentiment","description":"Entity-level financial sentiment; macro-F1. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/datasets/yixuantt/FinEntity","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Finance","Sentiment","Text"]},"finqa-128k":{"id":"finqa-128k","name":"FinQA 128k","topic":"Long-context financial QA","description":"Long-context (128k) variant of FinQA: analytics questions over financial reports.","url":"https://huggingface.co/yandex/AliceAI-Foundation-80B-A3B-Base","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long context","Text"]},"finsearchcomp":{"id":"finsearchcomp","name":"FinSearchComp","topic":"Financial retrieval","description":"Financial search and comprehension benchmark measuring retrieval grounded reasoning over financial content.","url":"https://randomtutu.github.io/FinSearchComp/","stars":"","updated_at":"","leaderboard":"https://randomtutu.github.io/FinSearchComp/","github":"","paper":"https://arxiv.org/pdf/2509.13160","data":"https://huggingface.co/datasets/ByteSeedXpert/FinSearchComp","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Reasoning","Text"]},"finsearchcomp-cn":{"id":"finsearchcomp-cn","name":"FinSearchComp-CN","topic":"Financial retrieval (Chinese)","description":"Chinese financial search and comprehension benchmark measuring retrieval-grounded reasoning over regional financial content.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","data":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Finance","Reasoning","Text"]},"finsearchcomp-t2t3":{"id":"finsearchcomp-t2t3","name":"FinSearchComp (T2&T3)","topic":"Finance search","description":"Finance search competition tasks (tracks T2 and T3).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Search","Text"]},"flame-react-eval":{"id":"flame-react-eval","name":"Flame-React-Eval","topic":"Frontend coding","description":"Front-end React coding tasks and evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Web","Text"]},"fleurs":{"id":"fleurs","name":"FLEURS","topic":"Speech recognition","description":"FLEURS multilingual automatic speech recognition benchmark; word error rate, lower is better.","url":"https://huggingface.co/datasets/google/fleurs","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"wer","higher_is_better":false,"classification":["Audio","ASR"]},"flickr30k":{"id":"flickr30k","name":"Flickr30K Captions","topic":"Image captioning","description":"Flickr30K image captioning (Karpathy test split, CIDEr).","url":"https://shannon.cs.illinois.edu/DenotationGraph/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://aclanthology.org/Q14-1006/","modality":"visual","metric":"cider","higher_is_better":true,"classification":["Captioning","Multimodal","Vision"]},"flores":{"id":"flores","name":"Flores","topic":"Machine translation (multilingual)","description":"FLORES multilingual translation benchmark.","url":"https://github.com/facebookresearch/flores","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/facebookresearch/flores","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Translation","Multilingual","Text"]},"flteval":{"id":"flteval","name":"FLTEval","topic":"Formal proof engineering","description":"Mistral benchmark built from real pull requests to the Fermat's Last Theorem Lean repository: complete all formal proofs and new definitions per PR.","url":"https://mistral.ai/news/leanstral","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Formal Proofs","Coding","Text"]},"forecastbench-decision":{"id":"forecastbench-decision","name":"ForecastBench (decision)","topic":"Forecasting","description":"ForecastBench questions answered as probabilities; Brier score x100 (lower is better). Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://www.forecastbench.org/","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"brier","higher_is_better":false,"classification":["Forecasting","Calibration","Text"]},"forte":{"id":"forte","name":"FORTE","topic":"General agent","description":"AGI-Eval FORTE general agentic task benchmark.","url":"https://github.com/AGI-Eval-Official/FORTE","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"fortress-adversarial":{"id":"fortress-adversarial","name":"FORTRESS (Adversarial)","topic":"Safety","description":"Scale AI FORTRESS adversarial prompts (CBRNE, political violence, crime); safety score.","url":"https://arxiv.org/abs/2506.14922","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"fortress-benign":{"id":"fortress-benign","name":"FORTRESS (Benign)","topic":"Safety (over-refusal)","description":"Scale AI FORTRESS benign counterpart prompts; share answered without over-refusal.","url":"https://arxiv.org/abs/2506.14922","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"fox-page-cn":{"id":"fox-page-cn","name":"Fox-Page-cn","topic":"Document OCR (Chinese)","description":"Fox Page benchmark evaluating OCR accuracy and layout understanding on Chinese document pages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Chinese","Document","OCR","Vision"]},"fox-page-en":{"id":"fox-page-en","name":"Fox-Page-en","topic":"Document OCR (English)","description":"Fox Page benchmark evaluating OCR accuracy and layout understanding on English document pages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Document","OCR","Vision"]},"fr-bench-pdf2md":{"id":"fr-bench-pdf2md","name":"fr-bench-pdf2md","topic":"Document parsing","description":"French PDF-to-Markdown conversion benchmark (baseline, forms, graphics, handwritten, long tables, multi-column, tiny text); overall score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document Understanding","Vision"]},"frames":{"id":"frames","name":"FRAMES","topic":"Interactive reasoning","description":"Frame-based interactive reasoning and dialogue benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"freshqa":{"id":"freshqa","name":"FreshQA","topic":"Recency QA","description":"Question answering benchmark emphasizing up-to-date knowledge and recency.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Recency","Text"]},"frontier-cs":{"id":"frontier-cs","name":"Frontier-CS","topic":"Algorithmic / open-ended CS problems","description":"Frontier-CS agent track (188 tasks): open-ended computer-science optimization problems scored by verifiable feedback.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Reasoning"]},"frontierbench":{"id":"frontierbench","name":"FrontierBench","topic":"Agentic terminal tasks","description":"Successor to Terminal-Bench 2.1 (v0.1): 74 harder terminal tasks emphasizing science and engineering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"frontiercode":{"id":"frontiercode","name":"FrontierCode","topic":"Software engineering","description":"Cognition's agentic coding benchmark of 150 tasks from real PRs, graded on hidden tests plus code-quality rubric (Main set, v1.1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"frontiercode-diamond":{"id":"frontiercode-diamond","name":"FrontierCode (Diamond)","topic":"Software engineering","description":"Cognition's FrontierCode agentic coding benchmark, Diamond subset; patch correctness graded by held-out tests plus rubric.","url":"https://cognition.ai/blog/frontier-code","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"frontiercode-extended":{"id":"frontiercode-extended","name":"FrontierCode Extended","topic":"Software engineering","description":"Cognition FrontierCode 1.1 Extended set (score).","url":"https://cognition.com/frontiercode","stars":"","updated_at":"","leaderboard":"https://cognition.com/frontiercode","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"frontiercode-main":{"id":"frontiercode-main","name":"FrontierCode Main","topic":"Software engineering","description":"Cognition FrontierCode 1.1 Main: coding-agent changes graded on correctness and mergeability (tests, scope, style, conventions).","url":"https://cognition.com/frontiercode","stars":"","updated_at":"","leaderboard":"https://cognition.com/frontiercode","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"frontierfinance":{"id":"frontierfinance","name":"FrontierFinance","topic":"Finance","description":"Professional finance benchmark combining quantitative reasoning with analysis of financial data and documents (reported in Step 5 Preview release).","url":"https://www.stepfun.com/step-5-preview","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Agents","Text"]},"frontiermath":{"id":"frontiermath","name":"FrontierMath","topic":"Math (frontier)","description":"Frontier mathematics benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"frontierscience":{"id":"frontierscience","name":"FrontierScience","topic":"Science reasoning","description":"Frontier-level scientific reasoning and QA benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Reasoning","QA","Text"]},"frontierscience-olympiad":{"id":"frontierscience-olympiad","name":"FrontierScience Olympiad","topic":"Science reasoning (olympiad)","description":"Olympiad-level problems from the FrontierScience benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Reasoning","Text"]},"frontierscience-research":{"id":"frontierscience-research","name":"FrontierScience Research","topic":"Science reasoning (research)","description":"Research-level problems from the FrontierScience benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Reasoning","Text"]},"frontierswe":{"id":"frontierswe","name":"FrontierSWE","topic":"Software engineering","description":"Proximal's FrontierSWE benchmark of frontier-difficulty software engineering tasks; dominance score (snapshot-dependent).","url":"https://www.proximal.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"frontiervep":{"id":"frontiervep","name":"FrontierVEP","topic":"Genomics","description":"Single-base variant-effect prediction benchmark.","url":"https://huggingface.co/HuggingFaceBio/Carbon-X","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","STEM","Text"]},"frontierxrd":{"id":"frontierxrd","name":"FrontierXRD","topic":"Materials science (XRD analysis)","description":"Periodic Labs benchmark of 134 hard X-ray diffraction analysis problems; success rate.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Agents","Text"]},"fsc-147":{"id":"fsc-147","name":"FSC-147","topic":"Few-shot counting","description":"Few-shot counting benchmark across 147 categories.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Counting","Multimodal","Vision"]},"fullstackbench":{"id":"fullstackbench","name":"FullStackBench","topic":"Full-stack development","description":"End-to-end web/app development tasks and evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text","Web"]},"fullstackbench-zh":{"id":"fullstackbench-zh","name":"FullStackBench (zh)","topic":"Full-stack development","description":"Chinese language full-stack development tasks and evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text","Web"]},"futureomni":{"id":"futureomni","name":"FutureOmni","topic":"Video forecasting","description":"FutureOmni: omni-modal (audio-visual) future event forecasting benchmark from video.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Audio","Reasoning"]},"gaia":{"id":"gaia","name":"GAIA","topic":"General AI tasks","description":"Comprehensive benchmark for agentic tasks.","url":"https://github.com/GAIA-benchmark/GAIA","stars":"","updated_at":"","leaderboard":"https://gaia-benchmark.github.io/leaderboard/","github":"https://github.com/GAIA-benchmark/GAIA","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"gaia-no-file":{"id":"gaia-no-file","name":"GAIA (no file)","topic":"General AI tasks","description":"GAIA benchmark subset without file inputs.","url":"https://github.com/GAIA-benchmark/GAIA","stars":"","updated_at":"","leaderboard":"https://gaia-benchmark.github.io/leaderboard/","github":"https://github.com/GAIA-benchmark/GAIA","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"gaia-text-only":{"id":"gaia-text-only","name":"GAIA (Text Only)","topic":"General AI tasks","description":"GAIA benchmark subset restricted to text-only inputs.","url":"https://github.com/GAIA-benchmark/GAIA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"gaia2":{"id":"gaia2","name":"GAIA 2","topic":"General agent tasks","description":"Grounded agentic intelligence benchmark version 2 covering multi-tool tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"gandalf-ignore":{"id":"gandalf-ignore","name":"Gandalf Ignore","topic":"Safety / jailbreak","description":"Resistance to Gandalf-style prompt ignore attacks.","url":"https://huggingface.co/datasets/Lakera/gandalf_ignore_instructions","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2501.07927","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"gaokao-2023-en":{"id":"gaokao-2023-en","name":"GaoKao 2023 En","topic":"Math (exam)","description":"English translation of 2023 Chinese college entrance exam (Gaokao) math questions, used in Qwen2.5-Math / DeepSeekMath style evaluations.","url":"https://github.com/QwenLM/Qwen2.5-Math","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2409.12122","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Text"]},"gaokao-bench":{"id":"gaokao-bench","name":"GAOKAO-Bench","topic":"Chinese exams","description":"GAOKAO benchmark measuring Chinese college entrance exam performance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Exams","Text"]},"gaokao-math-2026":{"id":"gaokao-math-2026","name":"Gaokao Math 2026","topic":"Math exams","description":"Five 2026 Chinese Gaokao math exams (National I, II, Beijing, Shanghai, Tianjin); average score out of 150.","url":"https://huggingface.co/datasets/XHToken/gaokao_math_2026","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"points","higher_is_better":true,"classification":["Math","Text"]},"gaokao-mathcloze":{"id":"gaokao-mathcloze","name":"Gaokao-MathCloze","topic":"Chinese math (exam)","description":"Fill-in-the-blank math problems from Chinese Gaokao (AGIEval).","url":"https://github.com/ruixiangcui/AGIEval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2304.06364","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Exams","Math","Text"]},"gaokao-mathqa":{"id":"gaokao-mathqa","name":"Gaokao-MathQA","topic":"Chinese math (exam)","description":"Multiple-choice math problems from Chinese Gaokao (AGIEval).","url":"https://github.com/ruixiangcui/AGIEval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2304.06364","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Exams","Math","Text"]},"gdp-pdf":{"id":"gdp-pdf","name":"GDP.pdf","topic":"Document understanding","description":"Surge AI multimodal benchmark of 100 professional prompts over information-dense PDFs; strict pass rate.","url":"https://surgehq.ai/blog/gdp-pdf-can-100b-ai-models-master-the-documents-that-run-the-world","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","Vision"]},"gdpval":{"id":"gdpval","name":"GDPVal","topic":"General capability","description":"GDPVal benchmark evaluating broad general capabilities of LLMs across diverse tasks.","url":"https://huggingface.co/datasets/openai/gdpval","stars":"","updated_at":"","leaderboard":"https://openai.com/index/gdpval/","github":"","paper":"https://cdn.openai.com/pdf/d5eb7428-c4e9-4a33-bd86-86dd4bcf12ce/GDPval.pdf","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Multitask","Text"]},"gdpval-aa-elo":{"id":"gdpval-aa-elo","name":"GDPVal-AA Elo","topic":"Office tasks","description":"GDPVal Artificial Analysis Elo rating for office-style tasks.","url":"https://openai.com/index/gdpval/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"elo","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"gdpval-aa-v2-elo":{"id":"gdpval-aa-v2-elo","name":"GDPval-AA v2 Elo","topic":"Office tasks","description":"Artificial Analysis GDPval-AA v2 leaderboard Elo (re-based; not comparable to v1 GDPval-AA Elo).","url":"https://artificialanalysis.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"elo","higher_is_better":true,"classification":["Agents","Reasoning","Text"]},"gdpval-rubrics":{"id":"gdpval-rubrics","name":"GDPval (rubrics)","topic":"Professional knowledge work","description":"GDPval tasks scored pointwise against rubrics (not pairwise win rate), as reported by MiniMax.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"genebench":{"id":"genebench","name":"GeneBench","topic":"Scientific data analysis","description":"OpenAI eval of multi-step scientific data analysis in genetics and quantitative biology, with ambiguous or flawed data and hidden confounders.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Agents","Reasoning","Text"]},"genebench-pro":{"id":"genebench-pro","name":"GeneBench Pro","topic":"Biology agent tasks","description":"Long-horizon genomics and quantitative-biology analysis tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","STEM","Text"]},"general-tool-use":{"id":"general-tool-use","name":"General Tool Use","topic":"Tool use","description":"General tool-use benchmark covering web and API tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"genomeqa":{"id":"genomeqa","name":"GenomeQA","topic":"Genomics","description":"DNA sequence understanding QA (enhancers, splice sites, organism, histone marks, TF binding, motifs).","url":"https://huggingface.co/HuggingFaceBio/Carbon-X","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","STEM","Text"]},"geobench":{"id":"geobench","name":"GeoBench","topic":"Geospatial reasoning","description":"Geospatial visual-question answering and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Reasoning","Vision"]},"geobench1":{"id":"geobench1","name":"GeoBench1","topic":"Geospatial reasoning","description":"Geospatial visual QA and reasoning (set 1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Reasoning","Vision"]},"global-mgsm":{"id":"global-mgsm","name":"GlobalMGSM","topic":"Math (multilingual)","description":"Global multilingual grade school math word problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Multilingual","Text"]},"global-mmlu":{"id":"global-mmlu","name":"Global-MMLU","topic":"Multi-domain knowledge (global)","description":"Full Global-MMLU evaluation across diverse languages and regions.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"global-mmlu-lite":{"id":"global-mmlu-lite","name":"Global-MMLU-Lite","topic":"Multi-domain knowledge (global)","description":"Lightweight global variant of MMLU covering diverse languages and regions.","url":"https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"global-piqa":{"id":"global-piqa","name":"Global PIQA","topic":"Commonsense reasoning across 100 Languages and Cultures","description":"Physical commonsense reasoning benchmark spanning 100 languages and diverse cultural contexts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Reasoning","Text"]},"glue":{"id":"glue","name":"GLUE","topic":"Multi-task NLU","description":"General Language Understanding Evaluation.","url":"https://gluebenchmark.com/","stars":"","updated_at":"2025-09-08","leaderboard":"https://gluebenchmark.com/leaderboard","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"gmai-mmbench":{"id":"gmai-mmbench","name":"GMAI-MMBench","topic":"Medical multimodal understanding","description":"Comprehensive medical multimodal benchmark evaluating vision–language models on clinical image and text tasks (e.g., QA, report understanding, diagnosis cues).","url":"https://uni-medical.github.io/GMAI-MMBench.github.io/#2023xtuner","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/uni-medical/GMAI-MMBench","paper":"https://arxiv.org/abs/2408.03361","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multimodal","QA","Vision"]},"gorilla-api-bench":{"id":"gorilla-api-bench","name":"Gorilla Benchmark API Bench","topic":"Tool use","description":"Gorilla API Bench tool-use evaluation.","url":"https://gorilla.cs.berkeley.edu/benchmark","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Text"]},"gpqa":{"id":"gpqa","name":"GPQA","topic":"Graduate-level QA","description":"Graduate-level question answering evaluating advanced reasoning.","url":"https://gpqa.github.io/","stars":"406","updated_at":"","leaderboard":"","github":"https://github.com/idavidrein/gpqa","paper":"https://arxiv.org/abs/2311.12022","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"gpqa-diamond":{"id":"gpqa-diamond","name":"GPQA-diamond","topic":"Graduate-level QA","description":"Hard subset of GPQA (diamond level).","url":"https://gpqa.github.io/","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"gpqa-main":{"id":"gpqa-main","name":"GPQA Main","topic":"Graduate-level QA","description":"Main GPQA set assessing advanced graduate-level reasoning.","url":"https://gpqa.github.io/","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"gqa":{"id":"gqa","name":"GQA","topic":"Visual reasoning VQA","description":"Compositional question answering over real-world images (scene-graph based); test-dev balanced accuracy.","url":"https://cs.stanford.edu/people/dorarad/gqa/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1902.09506","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"graphwalks-128k":{"id":"graphwalks-128k","name":"GraphWalks-128k","topic":"Long-context reasoning","description":"Graph-based long-context reasoning benchmark with 128k context length.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-bfs":{"id":"graphwalks-bfs","name":"GraphWalks BFS","topic":"Long-context reasoning","description":"Graph traversal/GraphWalks benchmark (BFS variant) for long-context reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-bfs-1m":{"id":"graphwalks-bfs-1m","name":"GraphWalks BFS 1M","topic":"Long-context reasoning","description":"GraphWalks BFS long-context graph traversal at 1M context (F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-bfs-256k":{"id":"graphwalks-bfs-256k","name":"GraphWalks BFS 256K","topic":"Long-context reasoning","description":"GraphWalks BFS long-context graph traversal at 256K context (F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-bfs-256k-1m":{"id":"graphwalks-bfs-256k-1m","name":"GraphWalks BFS 256K-1M","topic":"Long-context reasoning","description":"GraphWalks BFS long-context graph traversal at 256K-1M context (F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-parents":{"id":"graphwalks-parents","name":"GraphWalks Parents","topic":"Long-context reasoning","description":"Graph traversal/GraphWalks benchmark (Parents variant) for long-context reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-parents-1m":{"id":"graphwalks-parents-1m","name":"GraphWalks Parents 1M","topic":"Long-context reasoning","description":"GraphWalks Parents long-context graph task at 1M context (F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-parents-256k":{"id":"graphwalks-parents-256k","name":"GraphWalks Parents 256K","topic":"Long-context reasoning","description":"GraphWalks Parents long-context graph traversal at 256K context (F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"graphwalks-parents-256k-1m":{"id":"graphwalks-parents-256k-1m","name":"GraphWalks Parents 256K-1M","topic":"Long-context reasoning","description":"GraphWalks Parents long-context graph task, 256K-1M context bucket (accuracy).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"gre-math-maj16":{"id":"gre-math-maj16","name":"GRE Math maj@16","topic":"Math (standardized tests)","description":"GRE quantitative section evaluated via majority voting over 16 samples.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"ground-ui-1k":{"id":"ground-ui-1k","name":"Ground-UI-1K","topic":"GUI grounding","description":"Accuracy on the Ground-UI-1K grounding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Grounding","Multimodal","Vision"]},"gsm-infinite-hard-128k":{"id":"gsm-infinite-hard-128k","name":"GSM-Infinite Hard (128K)","topic":"Math reasoning","description":"GSM-Infinite Hard benchmark at 128K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Long Context","Text"]},"gsm-infinite-hard-16k":{"id":"gsm-infinite-hard-16k","name":"GSM-Infinite Hard (16K)","topic":"Math reasoning","description":"GSM-Infinite Hard benchmark at 16K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Long Context","Text"]},"gsm-infinite-hard-32k":{"id":"gsm-infinite-hard-32k","name":"GSM-Infinite Hard (32K)","topic":"Math reasoning","description":"GSM-Infinite Hard benchmark at 32K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Long Context","Text"]},"gsm-infinite-hard-64k":{"id":"gsm-infinite-hard-64k","name":"GSM-Infinite Hard (64K)","topic":"Math reasoning","description":"GSM-Infinite Hard benchmark at 64K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Long Context","Text"]},"gsm-plus":{"id":"gsm-plus","name":"GSM-Plus","topic":"Math (grade-school, enhanced)","description":"Enhanced GSM-style grade-school math benchmark variant.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"gsm-symbolic":{"id":"gsm-symbolic","name":"GSM-Symbolic","topic":"Math reasoning","description":"Symbolic reasoning variant of GSM that tests algebraic manipulation and arithmetic with structured problems.","url":"https://github.com/kosukearai/gsm-symbolic","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/kosukearai/gsm-symbolic","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"gsm8k":{"id":"gsm8k","name":"GSM8K","topic":"Math (grade-school)","description":"Grade-school math word problems requiring multi-step reasoning.","url":"https://huggingface.co/datasets/openai/gsm8k","stars":"1322","updated_at":"","leaderboard":"https://llm-stats.com/benchmarks/gsm8k","github":"https://github.com/openai/grade-school-math","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"gsm8k-de":{"id":"gsm8k-de","name":"GSM8K (DE)","topic":"Math (grade-school, German)","description":"German translation of the GSM8K grade-school math word problems.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"comp_acc","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text","German"]},"gsm8k-decision":{"id":"gsm8k-decision","name":"GSM8K (decision)","topic":"Math (grade-school)","description":"GSM8K posed as choices among candidate answers; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/datasets/openai/gsm8k","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Decision","Text"]},"gsm8k-ja":{"id":"gsm8k-ja","name":"GSM8K (Japanese)","topic":"Math (grade-school, Japanese)","description":"Japanese translation of GSM8K grade-school math.","url":"https://huggingface.co/datasets/SakanaAI/gsm8k-ja-test_250-1319","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text","Japanese"]},"gsm8k-ko":{"id":"gsm8k-ko","name":"GSM8K-Ko","topic":"Math (grade-school, Korean)","description":"Korean translation of the GSM8K grade-school math word problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text","Korean"]},"gsm8k-platinum":{"id":"gsm8k-platinum","name":"GSM8K Platinum","topic":"Math (grade-school, hard)","description":"Harder subset/setting of GSM8K grade-school math problems.","url":"https://github.com/openai/grade-school-math","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"gso-benchmark":{"id":"gso-benchmark","name":"GSO Benchmark","topic":"Code generation","description":"LiveCodeBench GSO benchmark.","url":"https://livecodebench.github.io/gso.html","stars":"","updated_at":"","leaderboard":"https://gso-bench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"habermas-machine-decision":{"id":"habermas-machine-decision","name":"Habermas Machine (decision)","topic":"Deliberation / preference","description":"Predict group preference among candidate consensus statements; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/google-deepmind/habermas_machine","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Preference","Decision","Text"]},"hae-rae-bench":{"id":"hae-rae-bench","name":"HAE-RAE Bench","topic":"Korean language understanding","description":"Korean language understanding benchmark evaluating knowledge and reasoning.","url":"https://huggingface.co/datasets/HAERAE-HUB/HAE_RAE_BENCH","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Korean","Text"]},"hallusionbench":{"id":"hallusionbench","name":"HallusionBench","topic":"Multimodal hallucination","description":"Benchmark for evaluating hallucination tendencies in multimodal LLMs.","url":"https://hallusionbench.github.io/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/FuxiaoLiu/HallusionBench","paper":"https://arxiv.org/abs/2310.14566","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Hallucination","Multimodal","Vision"]},"harbor-index":{"id":"harbor-index","name":"Harbor-Index","topic":"Agentic coding","description":"Aggregate index of agentic terminal/coding tasks run through the Harbor evaluation framework.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"hardmultiqa":{"id":"hardmultiqa","name":"HardMultiQA","topic":"Russian factual knowledge","description":"Yandex 5-shot benchmark of hard multi-answer factual questions in Russian.","url":"https://huggingface.co/datasets/yandex/HardMultiQA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"harmbench":{"id":"harmbench","name":"HarmBench","topic":"Safety","description":"Harmfulness and safety compliance benchmark across a variety of risky prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"harmfulq":{"id":"harmfulq","name":"HarmfulQA","topic":"Safety","description":"Harmful question set testing models' ability to avoid unsafe answers.","url":"https://huggingface.co/datasets/d-llm/HarmfulQA","stars":"104","updated_at":"","leaderboard":"","github":"https://github.com/declare-lab/red-instruct","paper":"https://arxiv.org/abs/2308.09662","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"harvey-lab-aa":{"id":"harvey-lab-aa","name":"Harvey LAB-AA","topic":"Legal agent tasks","description":"Harvey's Legal Agent Benchmark as run by Artificial Analysis (LAB-AA): complex legal workflows score.","url":"https://artificialanalysis.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Legal","Text"]},"healthbench":{"id":"healthbench","name":"HealthBench","topic":"Medical QA","description":"Comprehensive medical knowledge and clinical reasoning benchmark across specialties and tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"healthbench-hard":{"id":"healthbench-hard","name":"HealthBench-Hard","topic":"Medical QA (hard)","description":"Challenging subset of HealthBench focusing on complex, ambiguous clinical cases.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"healthbench-hard-hallucinations":{"id":"healthbench-hard-hallucinations","name":"HealthBench-Hard Hallucinations","topic":"Medical hallucination safety","description":"Measures hallucination and unsafe medical advice under hard clinical scenarios.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Safety","Text"]},"healthbench-professional":{"id":"healthbench-professional","name":"HealthBench Professional","topic":"Medical QA","description":"525 physician-authored clinical conversations (consults, documentation, research) graded against rubric criteria.","url":"https://arxiv.org/abs/2604.27470","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Reasoning","Text"]},"hellaswag":{"id":"hellaswag","name":"HellaSwag","topic":"Commonsense reasoning","description":"Adversarial commonsense sentence completion.","url":"https://github.com/rowanz/hellaswag","stars":"220","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"hellaswag-de":{"id":"hellaswag-de","name":"HellaSwag (DE)","topic":"Commonsense reasoning (German)","description":"German translation of the HellaSwag commonsense benchmark.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"norm_log_acc","higher_is_better":true,"classification":["Reasoning","Text","German"]},"hellaswag-multilingual":{"id":"hellaswag-multilingual","name":"HellaSwag Multilingual","topic":"Commonsense (multilingual)","description":"Multilingual variant of HellaSwag across multiple languages.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"helmet-longqa":{"id":"helmet-longqa","name":"HELMET LongQA","topic":"Long-context QA","description":"Long-context subset of the HELMET benchmark focusing on grounded question answering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","QA","Text"]},"helpsteer2":{"id":"helpsteer2","name":"HelpSteer2","topic":"Response quality rating","description":"NVIDIA HelpSteer2 helpfulness/attribute rating, posed as decisions.","url":"https://huggingface.co/datasets/nvidia/HelpSteer2","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Preference","Text"]},"herobench":{"id":"herobench","name":"HeroBench","topic":"Long-horizon planning","description":"Benchmark for Long-Horizon Planning and Structured Reasoning in Virtual Worlds.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/pdf/2508.12782","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Planning","Reasoning","Text"]},"hhem-v2.1":{"id":"hhem-v2.1","name":"HHEM v2.1","topic":"Hallucination detection","description":"Hughes Hallucination Evaluation Model (Vectara) — lower is better.","url":"","stars":"","updated_at":"","leaderboard":"https://github.com/vectara/hallucination-leaderboard","github":"https://github.com/vectara/hallucination-leaderboard","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Safety","Text"]},"hiddenmath":{"id":"hiddenmath","name":"HiddenMath","topic":"Math reasoning","description":"Mathematical reasoning benchmark referenced in recent model cards.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"hiertext-f1":{"id":"hiertext-f1","name":"HierText (F1)","topic":"OCR","description":"HierText text grounding and recognition, F1 mean.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","OCR"]},"hipho":{"id":"hipho","name":"HiPhO","topic":"Physics olympiad (multimodal)","description":"High-school physics olympiad problems with diagrams for evaluating multimodal physics reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Reasoning","Science","Vision"]},"hitab":{"id":"hitab","name":"HiTab","topic":"Table QA","description":"Hierarchical table QA (MMTab subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"hlab-vals-1.1":{"id":"hlab-vals-1.1","name":"HLAB-Vals-1.1","topic":"Legal agent tasks","description":"Vals AI run of the Harvey Legal Agent Benchmark: legal work using documents, spreadsheets, presentations and file-system tools.","url":"https://www.vals.ai/benchmarks/hlab","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Legal","Text"]},"hle":{"id":"hle","name":"HLE","topic":"Multi-domain reasoning","description":"Challenging LLMs at the frontier of human knowledge.","url":"https://github.com/centerforaisafety/hle","stars":"1085","updated_at":"","leaderboard":"https://agi.safe.ai/","github":"","paper":"","modality":"text","metric":"avg@3","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"hle-diamond":{"id":"hle-diamond","name":"HLE-Diamond","topic":"Multidisciplinary reasoning","description":"Hard 'Diamond' subset of Humanity's Last Exam, evaluated without tools.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Reasoning"},"hle-multichoice":{"id":"hle-multichoice","name":"HLE-MultiChoice","topic":"Advanced reasoning","description":"The 513 text-only original multiple-choice questions of Humanity's Last Exam, posed as one choice among the lettered answers.","url":"https://artificialanalysis.ai/evaluations/hle-multichoice","stars":"","updated_at":"","leaderboard":"https://artificialanalysis.ai/evaluations/hle-multichoice","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Knowledge","Text"]},"hle-overconfidence":{"id":"hle-overconfidence","name":"HLE Overconfidence","topic":"Overconfidence / safety","description":"Overconfidence rate derived from Humanity's Last Exam evaluations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Calibration","Text"]},"hle-text-only":{"id":"hle-text-only","name":"HLE (Text Only)","topic":"Advanced reasoning","description":"Humanity's Last Exam benchmark restricted to text-only inputs.","url":"https://github.com/centerforaisafety/hle","stars":"1085","updated_at":"","leaderboard":"https://agi.safe.ai/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"hle-verified":{"id":"hle-verified","name":"HLE-Verified","topic":"Multi-domain reasoning","description":"Verified and revised version of Humanity's Last Exam (HLE) with component-wise verification protocol.","url":"https://huggingface.co/datasets/skylenage/HLE-Verified","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"hle-vl":{"id":"hle-vl","name":"HLE-VL","topic":"Holistic language evaluation (vision-language)","description":"Vision-language HLE benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["General","Vision","Text"]},"hle-with-tools":{"id":"hle-with-tools","name":"HLE (With Tools)","topic":"Tool-augmented reasoning","description":"Humanity's Last Exam benchmark evaluated with tool access.","url":"https://github.com/centerforaisafety/hle","stars":"1085","updated_at":"","leaderboard":"https://agi.safe.ai/","github":"","paper":"","modality":"text","metric":"avg@3","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Tools","Text"]},"hmmt":{"id":"hmmt","name":"HMMT","topic":"Math (competition)","description":"Harvard–MIT Mathematics Tournament problems.","url":"https://hmmt.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"hmmt-2025":{"id":"hmmt-2025","name":"HMMT 2025","topic":"Math (competition)","description":"Harvard–MIT Mathematics Tournament 2025 problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"hmmt-feb-2025":{"id":"hmmt-feb-2025","name":"HMMT Feb 2025","topic":"Math (competition)","description":"Harvard–MIT Mathematics Tournament February 2025 problems.","url":"https://hmmt.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"hmmt-feb-2026":{"id":"hmmt-feb-2026","name":"HMMT Feb 2026","topic":"Math (competition)","description":"Harvard–MIT Mathematics Tournament February 2026 problems.","url":"https://hmmt.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"hmmt-nov-2025":{"id":"hmmt-nov-2025","name":"HMMT Nov 2025","topic":"Math (competition)","description":"Harvard–MIT Mathematics Tournament November 2025 problems.","url":"https://hmmt.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"home-appliance-simulator-decision":{"id":"home-appliance-simulator-decision","name":"Home appliance simulator (decision)","topic":"Device control","description":"Multi-step home appliance control decisions; case exact accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/spaces/multimodalart/jev-decision-index","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Decision","Text"]},"hotpotqa":{"id":"hotpotqa","name":"HotpotQA","topic":"Multi-hop QA","description":"Explainable multi-hop QA with supporting facts.","url":"https://hotpotqa.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"hover":{"id":"hover","name":"HoVer","topic":"Claim verification","description":"Multi-hop claim verification over Wikipedia introductions; accuracy (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/hover-nlp/hover","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Fact-checking","Text"]},"hrbench4k":{"id":"hrbench4k","name":"HRBench 4K","topic":"Hallucination robustness","description":"Hallucination robustness benchmark with 4K token contexts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Robustness","Text"]},"hrbench8k":{"id":"hrbench8k","name":"HRBench 8K","topic":"Hallucination robustness","description":"Hallucination robustness benchmark with 8K token contexts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Robustness","Text"]},"hrm8k":{"id":"hrm8k","name":"HRM8K","topic":"Korean reasoning","description":"8k-question Korean reasoning and knowledge benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Korean","Reasoning","Text"]},"hss":{"id":"hss","name":"HSS","topic":"Scene understanding","description":"HSS scene-understanding benchmark over images and video.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"humaneval":{"id":"humaneval","name":"HumanEval","topic":"Code generation","description":"Python synthesis problems evaluated by unit tests.","url":"https://github.com/openai/human-eval","stars":"2916","updated_at":"2025-09-08","leaderboard":"https://llmdb.com/benchmarks/humaneval","github":"https://github.com/openai/human-eval","paper":"https://arxiv.org/abs/2107.03374","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"humaneval-fim":{"id":"humaneval-fim","name":"HumanEval-FIM","topic":"Code infilling","description":"Fill-in-the-middle variant of HumanEval measuring code infilling.","url":"https://github.com/openai/human-eval-infilling","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2207.14255","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"humaneval-plus":{"id":"humaneval-plus","name":"HumanEval+","topic":"Code generation","description":"Extended HumanEval with more tests.","url":"https://github.com/evalplus/evalplus","stars":"1577","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"humaneval-v":{"id":"humaneval-v","name":"HumanEval-V","topic":"Code generation (vision)","description":"HumanEval variant with visual programming prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Code","Multimodal","Vision"]},"humaneval-x":{"id":"humaneval-x","name":"HumanEval-X","topic":"Code generation (multilingual)","description":"Multilingual code generation benchmark extending HumanEval to multiple programming languages.","url":"https://github.com/THUDM/CodeGeeX","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/THUDM/CodeGeeX","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"humanref":{"id":"humanref","name":"HumanRef","topic":"Referring expressions","description":"Referring detection of people in images (HumanRef), F1 mean.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"humicroedit-decision":{"id":"humicroedit-decision","name":"Humicroedit (decision)","topic":"Humor judgement","description":"Pick the funnier micro-edited headline; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://www.cs.rochester.edu/u/nhossain/humicroedit.html","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Humor","Decision","Text"]},"hungarian-math-exam":{"id":"hungarian-math-exam","name":"Hungarian National HS Math Exam","topic":"Math (held-out exam)","description":"Hungarian national high-school mathematics finals (2023) graded by humans; held-out math reasoning test.","url":"https://huggingface.co/datasets/keirp/hungarian_national_hs_finals_exam","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Text"]},"hybridial":{"id":"hybridial","name":"HybriDial","topic":"Conversational QA","description":"Hybrid table+text grounded dialogue QA (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"hypersim":{"id":"hypersim","name":"Hypersim","topic":"3D scene understanding","description":"Hypersim benchmark for synthetic indoor scene understanding and reconstruction.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["3D","Multimodal","Vision"]},"icdar2015-f1":{"id":"icdar2015-f1","name":"ICDAR2015 (F1)","topic":"OCR","description":"ICDAR2015 text grounding and recognition, F1 mean.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","OCR"]},"ifbench":{"id":"ifbench","name":"IFBench","topic":"Instruction following","description":"Instruction-following benchmark measuring compliance and adherence.","url":"https://github.com/allenai/IFBench","stars":"70","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"ifeval":{"id":"ifeval","name":"IFEval","topic":"Instruction following","description":"Instruction following capability evaluation for LLMs.","url":"https://github.com/google-research/google-research/tree/master/ifeval","stars":"36312","updated_at":"","leaderboard":"","github":"https://github.com/google-research/google-research/tree/master/instruction_following_eval","paper":"https://arxiv.org/abs/2311.07911","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"ifeval-code":{"id":"ifeval-code","name":"IFEval-Code","topic":"Instruction following (code)","description":"Instruction following evaluation for code generation tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Instruction Following","Text"]},"ifeval-strict-prompt":{"id":"ifeval-strict-prompt","name":"IFEval (Strict Prompt)","topic":"Instruction following","description":"IFEval strict prompt-level accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"ifstruct":{"id":"ifstruct","name":"IFStruct","topic":"Structured output","description":"Instruction-following benchmark for structured outputs (format/schema-constrained generation).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"imagemining":{"id":"imagemining","name":"ImageMining","topic":"Multimodal tool use","description":"Multimodal tool-use benchmark of mining information from images with tools.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision","Tools"]},"img-qa-avg":{"id":"img-qa-avg","name":"Image QA Average","topic":"Image QA (aggregate)","description":"Average of single-image visual question answering benchmarks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"imo-2025":{"id":"imo-2025","name":"IMO 2025","topic":"Math olympiad (proofs)","description":"2025 International Mathematical Olympiad (6 proof problems), graded with MathArena methodology.","url":"https://matharena.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"imo-answerbench":{"id":"imo-answerbench","name":"IMO AnswerBench","topic":"Math (competition)","description":"Evaluates free-form solutions to International Mathematical Olympiad problems using expert-style grading rubrics.","url":"https://imobench.github.io/","stars":"","updated_at":"","leaderboard":"https://imobench.github.io/#leaderboard","github":"https://github.com/google-deepmind/superhuman/tree/main/imobench","paper":"https://arxiv.org/abs/2511.01846","data":"https://github.com/google-deepmind/superhuman/tree/main/imobench","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"implicitmembench":{"id":"implicitmembench","name":"ImplicitMemBench","topic":"Long-term memory","description":"Tests implicit memory: whether agents apply previously seen preferences and procedures without being explicitly asked to recall them.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"impossible-exploitgym":{"id":"impossible-exploitgym","name":"Impossible ExploitGym","topic":"Safety","description":"ExploitGym tasks with no valid exploit; measures honest recognition of impossible tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Security","Text"]},"include":{"id":"include","name":"INCLUDE","topic":"Inclusiveness / bias","description":"Evaluates inclusive language use and bias mitigation in model outputs.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"include-v2":{"id":"include-v2","name":"INCLUDE V2","topic":"Inclusiveness / bias","description":"Updated INCLUDE benchmark with broader coverage and revised metrics.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"indicifeval-ground":{"id":"indicifeval-ground","name":"IndicIFEval (ground)","topic":"Indic languages","description":"Natively written Indic instruction-following prompts; prompt-level strict.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"indicifeval-trans":{"id":"indicifeval-trans","name":"IndicIFEval (translated)","topic":"Indic languages","description":"IFEval-style instruction following, translated into Indic languages; prompt-level strict.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"indicmmlu-pro":{"id":"indicmmlu-pro","name":"IndicMMLU-Pro","topic":"Indic languages","description":"MMLU-Pro translated/adapted to Indic languages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"indicsafe5":{"id":"indicsafe5","name":"IndicSafe5","topic":"Indic languages","description":"Safety benchmark of harmful/benign prompts across Indic languages; accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"indommlu":{"id":"indommlu","name":"IndoMMLU","topic":"Indonesian knowledge","description":"Indonesian multi-task exam questions from primary school to university entrance level.","url":"https://huggingface.co/datasets/indolem/IndoMMLU","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2310.04928","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Text"]},"infoqa":{"id":"infoqa","name":"InfoQA","topic":"Information-seeking QA","description":"Information retrieval question answering benchmark evaluating factual responses.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"information-extraction":{"id":"information-extraction","name":"Information Extraction","topic":"Information extraction","description":"Information extraction benchmark for economically valuable fields.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Text"]},"information-processing":{"id":"information-processing","name":"Information Processing","topic":"Information processing","description":"Information processing benchmark for economically valuable tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Text"]},"infotabs":{"id":"infotabs","name":"InfoTabs","topic":"Table NLI","description":"Natural language inference over infobox tables (MMTab subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"infovqa":{"id":"infovqa","name":"InfoVQA","topic":"Infographic VQA","description":"Visual question answering over infographics requiring reading, counting, and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Multimodal","QA","Reasoning","Vision"]},"inscit":{"id":"inscit","name":"INSCIT","topic":"Conversational QA","description":"Information-seeking conversations with mixed-initiative (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"intention-recognition":{"id":"intention-recognition","name":"Intention Recognition","topic":"Intent recognition","description":"Intent recognition benchmark for practical applications.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Text"]},"intphys-2":{"id":"intphys-2","name":"IntPhys 2","topic":"Intuitive physics","description":"Intuitive physics reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Physics","Reasoning","Vision"]},"inverse-ifeval":{"id":"inverse-ifeval","name":"Inverse IFEval","topic":"Instruction following (inverse)","description":"Inverse instruction-following evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"ioi":{"id":"ioi","name":"IOI","topic":"Reasoning","description":"Indirect object identification benchmark probing compositional reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"ioi-2025":{"id":"ioi-2025","name":"IOI 2025","topic":"Competitive programming","description":"International Olympiad in Informatics 2025 problem set; total score out of 600 points.","url":"https://ioi2025.bo/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"points","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"ipho-2025-theory":{"id":"ipho-2025-theory","name":"IPhO 2025 (Theory)","topic":"Physics olympiad","description":"Theory problems of the 2025 International Physics Olympiad.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Science"},"isarcasmeval":{"id":"isarcasmeval","name":"iSarcasmEval","topic":"Sarcasm detection","description":"SemEval-2022 iSarcasmEval track A (English) sarcasm F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/iabufarha/iSarcasmEval","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Classification","Text"]},"isl-osl-8k-16k":{"id":"isl-osl-8k-16k","name":"ISL/OSL 8k/16k","topic":"Throughput","description":"Relative throughput on ISL/OSL 8k/16k context workloads.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"relative_throughput","higher_is_better":true,"classification":["Performance","Throughput","Text"]},"jamc-qa":{"id":"jamc-qa","name":"JamC-QA","topic":"Japanese knowledge QA","description":"Japanese-specific knowledge multiple-choice QA (culture, customs, regional, law, medicine) from the Swallow evaluation suite.","url":"https://swallow-llm.github.io/leaderboard/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Japanese","Text"]},"jculture":{"id":"jculture","name":"JCulture","topic":"Japanese culture knowledge","description":"Japanese culture knowledge QA benchmark (as reported by Liquid AI).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text","Japanese"]},"jec-qa":{"id":"jec-qa","name":"JEC-QA","topic":"Legal QA (Chinese)","description":"Chinese National Judicial Examination multiple-choice QA.","url":"https://jecqa.thunlp.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1911.12011","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Legal","Text"]},"jevbench":{"id":"jevbench","name":"JevBench (Benchmark Heaven)","topic":"Typed decisions","description":"Benchmark Heaven JevBench for typed probabilistic decision models; score is the chance-corrected Intelligence (typed decision accuracy) axis.","url":"https://benchmarkheaven.com/jev-models","stars":"","updated_at":"","leaderboard":"https://benchmarkheaven.com/jev-models","github":"https://github.com/fstandhartinger/jevbench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Decision","Text"]},"jevbench-public":{"id":"jevbench-public","name":"JevBench public set","topic":"Typed decisions","description":"Accuracy on JevBench's 231 public items (easy, original/standard and hard tiers).","url":"https://github.com/fstandhartinger/jevbench","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/fstandhartinger/jevbench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Decision","Text"]},"jevbench-public-easy":{"id":"jevbench-public-easy","name":"JevBench public (easy)","topic":"Typed decisions","description":"Accuracy on the 48-item easy tier of JevBench's public set.","url":"https://github.com/fstandhartinger/jevbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Decision","Text"]},"jevbench-public-hard":{"id":"jevbench-public-hard","name":"JevBench public (hard)","topic":"Typed decisions","description":"Accuracy on the hard tier of JevBench's public set.","url":"https://github.com/fstandhartinger/jevbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Decision","Text"]},"jevbench-public-original":{"id":"jevbench-public-original","name":"JevBench public (original/standard)","topic":"Typed decisions","description":"Accuracy on the 72-item original (standard) tier of JevBench's public set.","url":"https://github.com/fstandhartinger/jevbench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Decision","Text"]},"jfbench":{"id":"jfbench","name":"JFBench","topic":"Instruction following (Japanese)","description":"Japanese instruction-following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text","Japanese"]},"jgpqa":{"id":"jgpqa","name":"JGPQA","topic":"Graduate-level QA (Japanese)","description":"Japanese translation of GPQA graduate-level science QA.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Reasoning","Text","Japanese"]},"jhumaneval":{"id":"jhumaneval","name":"JHumanEval","topic":"Code (Japanese)","description":"Japanese-prompt translation of HumanEval code generation benchmark.","url":"https://github.com/KuramitsuLab/jhuman-eval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Coding","Japanese","Text"]},"jhumaneval-plus":{"id":"jhumaneval-plus","name":"JHumanEval+","topic":"Code (Japanese)","description":"Japanese-prompt HumanEval+ code generation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text","Japanese"]},"jmmlu":{"id":"jmmlu","name":"JMMLU","topic":"Multi-domain knowledge (Japanese)","description":"Japanese Massive Multitask Language Understanding benchmark (translated and culture-specific MMLU questions).","url":"https://github.com/nlp-waseda/JMMLU","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text","Japanese"]},"jobbench":{"id":"jobbench","name":"JobBench","topic":"Professional tool use","description":"Agentic benchmark of professional, job-like tasks requiring tool use.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Agents"},"jobbench-easy":{"id":"jobbench-easy","name":"JobBench (Easy)","topic":"Professional job tasks","description":"JobBench easy split of realistic professional job tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"judgebench":{"id":"judgebench","name":"JudgeBench","topic":"LLM-as-judge","description":"Pairwise response-correctness judgement benchmark for LLM judges.","url":"https://github.com/ScalerLab/JudgeBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2410.12784","modality":"text","metric":"percent","higher_is_better":true,"classification":["Judge","Text"]},"judgemark-v2-1":{"id":"judgemark-v2-1","name":"JudgeMark v2.1","topic":"LLM judging ability","description":"A benchmark measuring LLM judging ability.","url":"","stars":"","updated_at":"","leaderboard":"https://eqbench.com/judgemark-v2.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Judging","Meta-Eval","Text"]},"kernelbench-hard":{"id":"kernelbench-hard","name":"KernelBench Hard","topic":"GPU kernel optimization","description":"KernelBench Hard: agents write optimized GPU kernels; score relative to hardware peak throughput.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"kernelgen-1p":{"id":"kernelgen-1p","name":"KernelGen 1P","topic":"AI research automation","description":"OpenAI internal eval (GPT-5.6 announcement): optimizing GPU kernels.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Research","Text"]},"kgc-safety":{"id":"kgc-safety","name":"KGC-Safety","topic":"Safety (Korean)","description":"Korean safety benchmark evaluating harmfulness and compliance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Korean","Safety","Text"]},"kimi-claw-24-7-bench":{"id":"kimi-claw-24-7-bench","name":"Kimi Claw 24/7 Bench","topic":"Long-horizon agents","description":"Moonshot in-house benchmark of persistent multi-day coworking tasks (17 scenarios, 610 evaluation points) run through the OpenClaw harness; average pass rate over 3 runs.","url":"https://huggingface.co/moonshotai/Kimi-K2.7-Code","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"kimi-code-bench-v2":{"id":"kimi-code-bench-v2","name":"Kimi Code Bench v2","topic":"Agentic coding","description":"Moonshot in-house coding-agent benchmark (Kimi Code Bench 2.0): realistic software engineering tasks across 10+ languages and production tech stacks.","url":"https://huggingface.co/moonshotai/Kimi-K2.7-Code","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"kimi-code-bench-v3-long-horizon":{"id":"kimi-code-bench-v3-long-horizon","name":"Kimi Code Bench 3.0 Long-horizon","topic":"Agentic coding","description":"Moonshot in-house long-horizon coding-agent benchmark (Kimi Code Bench 3.0, long-horizon split).","url":"https://www.kimi.ai/blog/kimi-k3-1-pro","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"kimi-hallu-bench":{"id":"kimi-hallu-bench","name":"Kimi Hallu Bench","topic":"Hallucination","description":"Moonshot in-house benchmark: percentage of factual claims containing errors, without tools (lower is better).","url":"https://www.kimi.ai/blog/kimi-k3-1-pro","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Text"]},"kimi-hallu-bench-hard-with-tools":{"id":"kimi-hallu-bench-hard-with-tools","name":"Kimi Hallu Bench Hard (With Tools)","topic":"Hallucination","description":"Moonshot in-house hard hallucination benchmark with tools: percentage of factual claims containing errors (lower is better).","url":"https://www.kimi.ai/blog/kimi-k3-1-pro","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Agents","Text"]},"kina":{"id":"kina","name":"KINA","topic":"Knowledge","description":"Knowledge benchmark of culturally grounded factual questions (reported in the Seed2.1 model card).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"kk-4-people":{"id":"kk-4-people","name":"KK-4 People","topic":"Working memory (4 people)","description":"Keep/kill working-memory benchmark with 4 people entities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Memory","Text"]},"kk-8-people":{"id":"kk-8-people","name":"KK-8 People","topic":"Working memory (8 people)","description":"Keep/kill working-memory benchmark with 8 people entities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Memory","Text"]},"kmmlu":{"id":"kmmlu","name":"KMMLU","topic":"Korean knowledge","description":"Korean Massive Multitask Language Understanding benchmark.","url":"https://huggingface.co/datasets/HAERAE-HUB/KMMLU","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Korean","Text"]},"kmmlu-pro":{"id":"kmmlu-pro","name":"KMMLU-Pro","topic":"Multilingual knowledge","description":"Korean Multilingual Massive Multitask Language Understanding Pro","url":"https://huggingface.co/datasets/HAERAE-HUB/KMMLU","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Text"]},"kmmlu-redux":{"id":"kmmlu-redux","name":"KMMLU-Redux","topic":"Multilingual knowledge","description":"Redux variant of KMMLU benchmark","url":"https://huggingface.co/datasets/HAERAE-HUB/KMMLU","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2507.08924","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Text"]},"ko-longbench":{"id":"ko-longbench","name":"Ko-LongBench","topic":"Korean long-context","description":"Long-context understanding benchmark in Korean.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Korean","Long Context","Text"]},"kobest":{"id":"kobest","name":"KoBEST","topic":"Korean understanding","description":"Korean Balanced Evaluation of Significant Tasks (BoolQ, COPA, WiC, HellaSwag, SentiNeg in Korean).","url":"https://huggingface.co/datasets/skt/kobest_v1","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Language","Korean","Text"]},"kobolt":{"id":"kobolt","name":"KoBALT","topic":"Korean knowledge","description":"Korean benchmark for knowledge and language understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Korean","Knowledge","Text"]},"komt-bench":{"id":"komt-bench","name":"KoMT-Bench","topic":"Korean chat ability","description":"Korean multi-turn chat evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"score","higher_is_better":true,"classification":["Korean","Text"]},"konet":{"id":"konet","name":"KoNET","topic":"Korean exams (multimodal)","description":"Korean National Educational Test benchmark (elementary to college entrance exams) with images.","url":"https://arxiv.org/abs/2502.15422","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Exams","Korean","Vision"]},"kor-bench":{"id":"kor-bench","name":"KOR-Bench","topic":"Reasoning","description":"Comprehensive reasoning benchmark spanning diverse domains and cognitive skills.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"korbench":{"id":"korbench","name":"KORBench","topic":"General reasoning","description":"Korean reasoning benchmark evaluating diverse reasoning capabilities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"kosimpleqa":{"id":"kosimpleqa","name":"KoSimpleQA","topic":"Korean QA","description":"Korean simple question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Korean","QA","Text"]},"ksm":{"id":"ksm","name":"KSM","topic":"Multilingual math","description":"Korean STEM and math benchmark","url":"","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Multilingual","Text"]},"kubricount-testb":{"id":"kubricount-testb","name":"KubriCount (testB)","topic":"Visual counting","description":"KubriCount testB object counting, exact-match accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Counting"]},"l-eval":{"id":"l-eval","name":"L-Eval","topic":"Long-context eval","description":"Long-context understanding and generation.","url":"https://github.com/OpenLMLab/LEval","stars":"389","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"labbench2":{"id":"labbench2","name":"LAB-Bench 2","topic":"Biology research","description":"LABBench2: biology real-world research tasks (macro-average over 11 sub-tasks: DbQA2, FigQA2-img/pdf, LitQA3, PatentQA, ProtocolQA2, SourceQuality, SuppQA2, TableQA2-img/pdf, TrialQA).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Science","Text"]},"lambada":{"id":"lambada","name":"LAMBADA","topic":"Language modeling","description":"Word prediction requiring broad context understanding.","url":"https://zenodo.org/record/2630551","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/pdf/1606.06031","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"latent-jailbreak":{"id":"latent-jailbreak","name":"LatentJailbreak","topic":"Safety / jailbreak","description":"Robustness to latent jailbreak adversarial techniques.","url":"","stars":"39","updated_at":"","leaderboard":"","github":"https://github.com/qiuhuachuan/latent-jailbreak","paper":"https://arxiv.org/pdf/2307.08487.pdf","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"lbv1-qa":{"id":"lbv1-qa","name":"LBV1-QA","topic":"Vision-language","description":"Vision-language QA benchmark v1.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"lbv2":{"id":"lbv2","name":"LBV2","topic":"Vision-language","description":"Vision-language benchmark v2.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Text","Vision"]},"leetcode-contest":{"id":"leetcode-contest","name":"LeetCode Contest","topic":"Code generation (competition)","description":"LeetCode weekly/biweekly contest problems (Jul 2023 - Jan 2024) collected by DeepSeek; pass@1.","url":"https://github.com/deepseek-ai/DeepSeek-Coder/tree/main/Evaluation/LeetCode","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2401.14196","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"legal-agent-benchmark":{"id":"legal-agent-benchmark","name":"Legal Agent Benchmark","topic":"Legal agents","description":"Harvey Legal Agent Benchmark, full public set.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Agents","Text"]},"legal-agent-benchmark-aa":{"id":"legal-agent-benchmark-aa","name":"Harvey LAB-AA","topic":"Legal agents","description":"Artificial Analysis run of Harvey's Legal Agent Benchmark (legal work products from source documents).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Agents"},"legal-agent-benchmark-held-out":{"id":"legal-agent-benchmark-held-out","name":"Legal Agent Benchmark (Held-Out)","topic":"Legal agents","description":"Harvey Legal Agent Benchmark, Harvey's held-out set.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Agents","Text"]},"legal-research-bench":{"id":"legal-research-bench","name":"Legal Research Bench","topic":"Legal research","description":"Vals AI Legal Research benchmark of agentic legal research tasks.","url":"https://www.vals.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Agents","Text"]},"legalbench":{"id":"legalbench","name":"LegalBench","topic":"Legal QA","description":"Suite of legal question answering and reasoning tasks.","url":"https://legalbench.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","QA","Text"]},"lexam":{"id":"lexam","name":"LEXam","topic":"Legal exams","description":"LEXam benchmark of law-school exam questions (open and multiple-choice) across jurisdictions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","QA","Text"]},"lexam-hard":{"id":"lexam-hard","name":"LEXam (hard)","topic":"Legal exams","description":"Hard subset of the LEXam legal-exam benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","QA","Text"]},"lexam-mcq4-en":{"id":"lexam-mcq4-en","name":"LEXam MCQ (4-choice, EN)","topic":"Legal exams","description":"LEXam legal-exam benchmark, English 4-option multiple-choice subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","QA","Text"]},"lifebench":{"id":"lifebench","name":"LIFEBench","topic":"Instruction following","description":"Length-based instruction-following evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"lifescibench":{"id":"lifescibench","name":"LifeSciBench","topic":"Life sciences","description":"Life-sciences research workflow benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["STEM","Reasoning","Text"]},"lingoqa":{"id":"lingoqa","name":"LingoQA","topic":"Driving scene QA","description":"Question answering benchmark for autonomous driving scene understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Autonomous Driving","Multimodal","QA","Vision"]},"livebench":{"id":"livebench","name":"LiveBench","topic":"General capability","description":"Continually updated capability benchmark across diverse tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["General","Text"]},"livebrowsecomp":{"id":"livebrowsecomp","name":"LiveBrowseComp","topic":"Agentic web browsing","description":"Live (continuously refreshed) BrowseComp-style benchmark of hard-to-find web information for deep-search agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Search","Text"]},"liveclaw":{"id":"liveclaw","name":"LiveClaw","topic":"Agentic assistant tasks","description":"Live benchmark of OpenClaw agent tasks executed in real environments.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"livecodebench":{"id":"livecodebench","name":"LiveCodeBench","topic":"Code generation","description":"Live coding and execution-based evaluation benchmark (v6 dataset).","url":"https://livecodebench.github.io/","stars":"","updated_at":"2025-09-09","leaderboard":"https://livecodebench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"livecodebench-ko":{"id":"livecodebench-ko","name":"LiveCodeBench-Ko","topic":"Code generation (Korean)","description":"Korean translation of LiveCodeBench.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Korean","Text"]},"livecodebench-pro":{"id":"livecodebench-pro","name":"LiveCodeBench Pro","topic":"Competitive coding problems from Codeforces, ICPC, and IOI","description":"LiveCodeBench Pro evaluates competitive programming performance across Codeforces, ICPC, and IOI contests. Elo rating, higher is better.","url":"https://livecodebench.github.io/","stars":"","updated_at":"","leaderboard":"https://livecodebench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"elo","higher_is_better":true,"classification":["Code","Text"]},"livecodebench-pro-25q2-easy":{"id":"livecodebench-pro-25q2-easy","name":"LCB Pro 25Q2 (Easy)","topic":"Code generation","description":"LiveCodeBench Pro 2025 Q2 easy subset.","url":"https://livecodebench.github.io/","stars":"","updated_at":"2025-06-30","leaderboard":"https://livecodebench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"livecodebench-pro-25q2-med":{"id":"livecodebench-pro-25q2-med","name":"LCB Pro 25Q2 (Med)","topic":"Code generation","description":"LiveCodeBench Pro 2025 Q2 medium subset.","url":"https://livecodebench.github.io/","stars":"","updated_at":"2025-06-30","leaderboard":"https://livecodebench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"livecodebench-v3":{"id":"livecodebench-v3","name":"LiveCodeBench v3","topic":"Code generation","description":"LiveCodeBench v3 snapshot measuring pass rates on streaming coding tasks.","url":"https://livecodebench.github.io/","stars":"","updated_at":"2025-11-20","leaderboard":"https://livecodebench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"livecodebench-v5":{"id":"livecodebench-v5","name":"LiveCodeBench v5 (2024.10-2025.02)","topic":"Code generation","description":"LiveCodeBench v5 snapshot covering Oct 2024-Feb 2025.","url":"https://livecodebench.github.io/","stars":"","updated_at":"2025-02-28","leaderboard":"https://livecodebench.github.io/leaderboard.html","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"livemathematicianbench":{"id":"livemathematicianbench","name":"LiveMathematicianBench","topic":"Research math","description":"Live benchmark of research-level mathematics problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"livemcp-101":{"id":"livemcp-101","name":"LiveMCP-101","topic":"Agent real-time eval","description":"A novel real-time evaluation framework and benchmark to stress‑test agents on complex, real‑world tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/pdf/2508.15760","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Real-time","Tools","Text"]},"livesports-3k":{"id":"livesports-3k","name":"LiveSports-3K","topic":"Sports video","description":"Live sports video understanding benchmark (3K).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Sports","Vision"]},"livesqlbench":{"id":"livesqlbench","name":"LiveSQLBench","topic":"Text-to-SQL","description":"LiveSQLBench (Base-Full v1): contamination-free text-to-SQL / database tasks over PostgreSQL databases, solved agentically.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"llava-bench-in-the-wild":{"id":"llava-bench-in-the-wild","name":"LLaVA-Bench (In-the-Wild)","topic":"Visual chat","description":"GPT-4-judged open-ended visual conversation, detailed description and complex reasoning on 60 in-the-wild images; relative score vs. GPT-4 reference.","url":"https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/haotian-liu/LLaVA","paper":"https://arxiv.org/abs/2304.08485","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"lmarena-text":{"id":"lmarena-text","name":"LMArena Text","topic":"Crowd eval (text)","description":"Chatbot Arena text evaluation (average win rate).","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"https://lmarena.ai/leaderboard/text","github":"","paper":"","modality":"text","metric":"elo","higher_is_better":true,"classification":["Text"]},"lmarena-vision":{"id":"lmarena-vision","name":"LMArena Vision","topic":"Crowd eval (vision)","description":"Chatbot Arena vision evaluation leaderboard (ELO ratings).","url":"","stars":"","updated_at":"","leaderboard":"https://lmarena.ai/leaderboard/vision","github":"","paper":"","modality":"visual","metric":"elo","higher_is_better":true,"classification":["Vision","Multimodal"]},"loca-bench":{"id":"loca-bench","name":"LOCA-Bench","topic":"Long-context agents","description":"LOCA-Bench long-context agent benchmark (environment description length 256K).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Long Context","Text"]},"local-agent-bench":{"id":"local-agent-bench","name":"Local Agent Bench","topic":"Tool calling judgment","description":"Tests whether small open-weight models can reliably decide when to call tools and when not to. Agent Score = (Action x 0.4) + (Restraint x 0.3) + (Wrong-Tool-Avoidance x 0.3).","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/MikeVeerman/tool-calling-benchmark","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Agents","Text"]},"locomo":{"id":"locomo","name":"LoCoMo","topic":"Long-term memory","description":"Very long-term conversational memory: QA over multi-session dialogues.","url":"https://snap-research.github.io/locomo/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2402.17753","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"logicvista":{"id":"logicvista","name":"LogicVista","topic":"Visual logical reasoning","description":"Visual logic and pattern reasoning tasks requiring compositional and spatial understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"logiqa":{"id":"logiqa","name":"LogiQA","topic":"Logical reasoning","description":"Reading comprehension with logical reasoning.","url":"https://github.com/lgw863/LogiQA-dataset","stars":"138","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"longbench":{"id":"longbench","name":"LongBench","topic":"Long-context eval","description":"Long-context understanding across tasks.","url":"https://github.com/THUDM/LongBench","stars":"957","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"longbench-v2":{"id":"longbench-v2","name":"longbench-v2","topic":"Long-context eval","description":"Next-generation LongBench v2 long-context evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2412.15204","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"longcodebench":{"id":"longcodebench","name":"LongCodeBench","topic":"Long-context code understanding","description":"Long-context code comprehension and repair benchmark (reported at 1M-token context).","url":"https://arxiv.org/abs/2505.07897","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Long Context","Text"]},"longfact-concepts":{"id":"longfact-concepts","name":"LongFact-Concepts","topic":"Factuality on open-source prompts","description":"Long-context factuality eval focused on conceptual statements; scored as 100 - hallucination rate, higher is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Safety","Text"]},"longfact-objects":{"id":"longfact-objects","name":"LongFact-Objects","topic":"Factuality on open-source prompts","description":"Long-context factuality eval focused on object/entity references; scored as 100 - hallucination rate, higher is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Hallucination","Safety","Text"]},"longmemeval":{"id":"longmemeval","name":"LongMemEval","topic":"Long-term memory","description":"Evaluates LLMs' long-term memory and retrieval over extended contexts and multi-session interactions.","url":"https://xiaowu0162.github.io/long-mem-eval/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/xiaowu0162/LongMemEval","paper":"https://arxiv.org/abs/2410.10813","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"longtext-bench-en":{"id":"longtext-bench-en","name":"LongText-Bench EN","topic":"Text rendering","description":"LongText-Bench English subset score for text rendering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"longtext-bench-zh":{"id":"longtext-bench-zh","name":"LongText-Bench ZH","topic":"Text rendering","description":"LongText-Bench Chinese subset score for text rendering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"longvideobench":{"id":"longvideobench","name":"LongVideoBench","topic":"Long video QA","description":"Long video understanding and QA benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Long Context","QA","Vision"]},"lpfqa":{"id":"lpfqa","name":"LPFQA","topic":"Finance QA","description":"Long-form financial question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","QA","Text"]},"lvbench":{"id":"lvbench","name":"LVBench","topic":"Video understanding","description":"Long video understanding benchmark (LVBench).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"lvis-det-f1":{"id":"lvis-det-f1","name":"LVIS Detection (F1)","topic":"Object detection","description":"LVIS object detection with an MLLM; F1 averaged over IoU 0.5:0.95.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"lvis-pointing":{"id":"lvis-pointing","name":"LVIS Pointing","topic":"Object pointing","description":"LVIS object pointing accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"m-ifeval-ja":{"id":"m-ifeval-ja","name":"M-IFEval (Japanese)","topic":"Instruction following (Japanese)","description":"Japanese split of M-IFEval multilingual instruction-following benchmark.","url":"https://arxiv.org/abs/2502.04688","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text","Japanese"]},"m3gia-cn":{"id":"m3gia-cn","name":"M3GIA (CN)","topic":"Chinese multimodal QA","description":"Chinese-language M3GIA benchmark covering grounded multimodal question answering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Multimodal","QA","Vision"]},"machiavelli":{"id":"machiavelli","name":"Machiavelli","topic":"Deception / safety","description":"Benchmark for deceptive or manipulative behavior in social interactions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Deception","Text"]},"makemesay":{"id":"makemesay","name":"MakeMeSay","topic":"Adversarial robustness","description":"Adversarial benchmark testing model robustness against manipulation attempts. Lower is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Text"]},"mantis":{"id":"mantis","name":"Mantis","topic":"Multimodal reasoning","description":"Multimodal reasoning and instruction following benchmark (Mantis).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"marena-hard":{"id":"marena-hard","name":"mArenaHard","topic":"Chat ability (multilingual)","description":"Multilingual variant of Arena-Hard evaluating chat quality across languages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"mars-bench":{"id":"mars-bench","name":"MARS-Bench","topic":"Instruction following","description":"Instruction-following benchmark with complex tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"mask":{"id":"mask","name":"MASK","topic":"Safety / red teaming","description":"Model behavior safety assessment via red-teaming scenarios.","url":"https://github.com/centerforaisafety/mask","stars":"","updated_at":"","leaderboard":"https://scale.com/leaderboard/mask","github":"https://github.com/centerforaisafety/mask","paper":"https://arxiv.org/html/2503.03750v1","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"massive-intent":{"id":"massive-intent","name":"MASSIVE intent","topic":"Intent classification","description":"Amazon MASSIVE intent classification (English).","url":"https://github.com/alexa/massive","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Classification","Text"]},"matbench":{"id":"matbench","name":"MatBench","topic":"Materials property prediction","description":"Materials property prediction benchmark for scientific AI models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Materials","Science","STEM","Vision"]},"math":{"id":"math","name":"MATH","topic":"Math (competition)","description":"Competition-level mathematics across algebra, geometry, number theory, combinatorics.","url":"https://github.com/hendrycks/math","stars":"1185","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2103.03874","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"math-100-ja":{"id":"math-100-ja","name":"MATH-100 (Japanese)","topic":"Math (Japanese)","description":"100 MATH problems translated to Japanese (Swallow evaluation suite).","url":"https://swallow-llm.github.io/leaderboard/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Japanese","Text"]},"math-ko":{"id":"math-ko","name":"MATH-Ko","topic":"Math (Korean)","description":"Korean translation of the MATH competition benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Korean","STEM","Text"]},"math-lvl5":{"id":"math-lvl5","name":"MATH Level 5","topic":"Math (competition)","description":"Level 5 subset of the MATH benchmark emphasizing the hardest competition-style problems.","url":"https://github.com/hendrycks/math","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"math500":{"id":"math500","name":"MATH500","topic":"Math reasoning","description":"500-problem slice of the MATH benchmark for challenging math reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"math500-es":{"id":"math500-es","name":"MATH500 (ES)","topic":"Math (multilingual)","description":"Spanish MATH500 benchmark","url":"","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Multilingual","Text"]},"math500-ja":{"id":"math500-ja","name":"MATH500 (Japanese)","topic":"Math (Japanese)","description":"Japanese translation of MATH500.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Multilingual","Text","Japanese"]},"matharena-apex":{"id":"matharena-apex","name":"MathArena Apex","topic":"Challenging Math Contest problems","description":"Challenging math contest problems from MathArena Apex benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Reasoning","Text"]},"matharena-apex-shortlist":{"id":"matharena-apex-shortlist","name":"MathArena Apex Shortlist","topic":"Mathematics","description":"Shortlist of candidate hard competition problems considered for MathArena Apex.","url":"https://matharena.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning"]},"mathbench-a":{"id":"mathbench-a","name":"MathBench-A","topic":"Math (multi-level)","description":"MathBench application split covering arithmetic, primary, middle, high school and college levels (overall score).","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/open-compass/MathBench","paper":"https://arxiv.org/abs/2405.12209","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"mathbench-zh":{"id":"mathbench-zh","name":"MathBench (ZH)","topic":"Math (Chinese)","description":"Chinese portion of MathBench as reported in the InternLM-Math paper.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/open-compass/MathBench","paper":"https://arxiv.org/abs/2405.12209","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Math","STEM","Text"]},"matholympiadbench":{"id":"matholympiadbench","name":"MathOlympiadBench","topic":"Formal theorem proving","description":"Olympiad-level math problems formalized in Lean 4; score = share of theorems proved (Pass@32 in LongCat-Flash-Prover report).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2603.21065","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"mathqa":{"id":"mathqa","name":"MathQA","topic":"Math (word problems)","description":"Math word problems requiring reasoning and operations.","url":"https://math-qa.github.io/","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"mathverse":{"id":"mathverse","name":"MathVerse","topic":"Math reasoning (multimodal)","description":"Visual math reasoning benchmark combining images and text across diverse mathematical tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Reasoning","STEM","Vision"]},"mathverse-mini":{"id":"mathverse-mini","name":"MathVerse-mini","topic":"Math reasoning (multimodal)","description":"Compact MathVerse split focusing on single-image math puzzles and visual reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Reasoning","STEM","Vision"]},"mathverse-vision":{"id":"mathverse-vision","name":"MathVerse-Vision","topic":"Math reasoning (multimodal)","description":"Multi-image visual mathematical reasoning tasks from the MathVerse ecosystem.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Reasoning","STEM","Vision"]},"mathvision":{"id":"mathvision","name":"MathVision","topic":"Math reasoning (multimodal)","description":"Visual math reasoning benchmark with problems that combine images (charts, diagrams) and text.","url":"https://mathllm.github.io/mathvision","stars":"","updated_at":"","leaderboard":"https://mathllm.github.io/mathvision/#leaderboard","github":"https://github.com/mathllm/MATH-V","paper":"https://proceedings.neurips.cc/paper_files/paper/2024/file/ad0edc7d5fa1a783f063646968b7315b-Paper-Datasets_and_Benchmarks_Track.pdf","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Charts","Document","Math","Multimodal","Reasoning","STEM","Vision"]},"mathvision-with-tools":{"id":"mathvision-with-tools","name":"MathVision (With Tools)","topic":"Math reasoning (multimodal)","description":"MathVision visual math benchmark evaluated with a Python code-interpreter tool.","url":"https://mathllm.github.io/mathvision","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Vision"]},"mathvista":{"id":"mathvista","name":"MathVista","topic":"Multimodal math reasoning","description":"Visual math reasoning across diverse tasks.","url":"https://mathvista.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Reasoning","STEM","Vision"]},"mathvista-mini":{"id":"mathvista-mini","name":"MathVista-Mini","topic":"Math reasoning (multimodal)","description":"Lightweight subset of MathVista for quick evaluation of visual mathematical reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Math","Multimodal","Reasoning","STEM","Vision"]},"mattools":{"id":"mattools","name":"MatTools","topic":"Materials science tool use","description":"Benchmark evaluating LLM agents on materials-science tool usage (pymatgen-based QA and code/tool generation tasks).","url":"https://arxiv.org/abs/2505.10852","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Science","Text"]},"maverix":{"id":"maverix","name":"MAVERIX","topic":"Audio-visual video understanding","description":"Benchmark requiring joint video and audio understanding (situational awareness, social sentiment).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Audio","Multimodal"]},"maxife":{"id":"maxife","name":"MAXIFE","topic":"Instruction following (multilingual)","description":"Multilingual instruction-following evaluation across English and multilingual original prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Multilingual","Text"]},"mbe-bar-exam":{"id":"mbe-bar-exam","name":"MBE Bar Exam","topic":"Legal exams","description":"Multistate Bar Examination (US bar exam) multiple-choice questions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","QA","Text"]},"mbpp":{"id":"mbpp","name":"MBPP","topic":"Code generation","description":"Short Python problems with hidden tests.","url":"https://huggingface.co/datasets/Muennighoff/mbpp","stars":"36312","updated_at":"","leaderboard":"https://llm-stats.com/benchmarks","github":"","paper":"https://arxiv.org/abs/2108.07732","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"mbpp-ko":{"id":"mbpp-ko","name":"MBPP-Ko","topic":"Code generation (Korean)","description":"Korean translation of MBPP code generation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Korean","Text"]},"mbpp-plus":{"id":"mbpp-plus","name":"MBPP+","topic":"Code generation","description":"Extended MBPP with more tests and stricter evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"mcp-atlas":{"id":"mcp-atlas","name":"MCP-Atlas","topic":"Agent evaluation","description":"Aggregate MCP agent benchmark covering tool-use and planning tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"mcp-universe":{"id":"mcp-universe","name":"MCP Universe","topic":"Agent evaluation","description":"Benchmarks multi-step tool-use agents across diverse task suites with a unified overall success metric.","url":"https://mcp-universe.github.io/","stars":"","updated_at":"","leaderboard":"https://mcp-universe.github.io/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Reasoning","Tool Use"]},"mcpmark":{"id":"mcpmark","name":"MCPMark","topic":"Agent tool-use (MCP)","description":"Benchmark for Model Context Protocol (MCP) agent tool-use.","url":"https://mcpmark.ai/","stars":"127","updated_at":"","leaderboard":"https://mcpmark.ai/","github":"https://github.com/eval-sys/mcpmark","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"mcpmark-verified":{"id":"mcpmark-verified","name":"MCPMark-Verified","topic":"Agent tool-use (MCP)","description":"Human-verified edition of MCPMark covering Notion, GitHub, Filesystem, Postgres and Playwright MCP servers.","url":"https://mcpmark.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tool use","Text"]},"mdolly":{"id":"mdolly","name":"mDolly","topic":"Instruction following (multilingual)","description":"Multilingual variant of the Dolly instruction-following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Multilingual","Text"]},"mdpbench":{"id":"mdpbench","name":"MDPBench","topic":"Document parsing","description":"Document parsing benchmark (reported in OvisOCR3 release).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"measurebench":{"id":"measurebench","name":"MeasureBench","topic":"Visual measurement reading","description":"Benchmark of reading measurement instruments (gauges, rulers, meters) from images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Perception"]},"medbullets-op4":{"id":"medbullets-op4","name":"Medbullets (4 options)","topic":"Medical QA","description":"USMLE Step 2/3-style clinical questions, 4-option version.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"medbullets-op5":{"id":"medbullets-op5","name":"Medbullets (5 options)","topic":"Medical QA","description":"USMLE Step 2/3-style clinical questions, 5-option version.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"medchembench-internal":{"id":"medchembench-internal","name":"MedChemBench (internal)","topic":"Medicinal chemistry","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: medicinal chemistry tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Science","Chemistry","Text"]},"medmcqa":{"id":"medmcqa","name":"MedMCQA","topic":"Medical exams","description":"Indian medical entrance exam (AIIMS/NEET PG) multiple-choice QA.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"medqa":{"id":"medqa","name":"MedQA","topic":"Medical exams","description":"US medical licensing exam-style multiple choice questions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Exams","Text"]},"medqa-mcmle":{"id":"medqa-mcmle","name":"MedQA (MCMLE)","topic":"Medical QA (Chinese)","description":"MedQA Mainland China Medical Licensing Exam subset (5 options).","url":"https://github.com/jind11/MedQA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2009.13081","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Medical","Text"]},"medxpertqa-mm":{"id":"medxpertqa-mm","name":"MedXpertQA-MM","topic":"Medical VQA","description":"Multimodal medical expert question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Multimodal","QA","Vision"]},"medxpertqa-text":{"id":"medxpertqa-text","name":"MedXpertQA (Text)","topic":"Medical QA","description":"Text-only subset of MedXpertQA expert-level medical multiple-choice questions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Health"},"meeseeks-zh":{"id":"meeseeks-zh","name":"Meeseeks-zh","topic":"Instruction following (Chinese)","description":"Chinese variant of the Meeseeks instruction-following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","Instruction Following","Text"]},"membench":{"id":"membench","name":"MemBench","topic":"Long-term memory","description":"Benchmark of factual and reflective memory of LLM-based agents across participation and observation scenarios.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2506.21605","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"memoryagentbench":{"id":"memoryagentbench","name":"MemoryAgentBench","topic":"Long-term memory","description":"Evaluates memory agents on accurate retrieval, test-time learning, long-range understanding and conflict resolution.","url":"https://github.com/HUST-AI-HYZ/MemoryAgentBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2507.05257","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"memorybench":{"id":"memorybench","name":"MemoryBench","topic":"Long-term memory","description":"Meta-reported benchmark of remembering and retrieving user context across conversations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Agents"},"metr":{"id":"metr","name":"METR","topic":"Long task benchmark","description":"METR evaluates AI agents on long-horizon coding and agentic tasks, measuring autonomous task completion time.","url":"https://metr.org","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"hours","higher_is_better":true,"classification":["Coding","Agentic","Long Task"]},"mewc":{"id":"mewc","name":"MEWC","topic":"Web comprehension","description":"Multi-page End-to-end Web Comprehension benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Web","Text"]},"mgsm":{"id":"mgsm","name":"MGSM","topic":"Math (multilingual)","description":"Multilingual grade school math word problems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"mhpp":{"id":"mhpp","name":"MHPP","topic":"Code generation","description":"Mostly Hard Python Problems: 210 hand-written function-level Python problems probing harder code-generation skills; pass@1.","url":"https://github.com/SparksofAGI/MHPP","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/SparksofAGI/MHPP","paper":"https://arxiv.org/abs/2405.11430","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"miabench":{"id":"miabench","name":"MIABench","topic":"Multimodal instruction following","description":"Multimodal instruction-following benchmark evaluating accuracy on complex image-text tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Multimodal","Vision"]},"microvqa":{"id":"microvqa","name":"MicroVQA","topic":"Biological microscopy","description":"Visual question answering benchmark for biological microscopy images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Biology","Science","Vision"]},"milu":{"id":"milu","name":"MILU","topic":"Indic languages","description":"Multi-task Indic Language Understanding benchmark (11 Indian languages + English), accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"mimic-cxr":{"id":"mimic-cxr","name":"MIMIC-CXR (classification)","topic":"Medical imaging","description":"Chest X-ray condition classification on MIMIC-CXR, macro F1 over top conditions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"mimo-niah-multi-128k":{"id":"mimo-niah-multi-128k","name":"NIAH-Multi 128K","topic":"Long-context QA","description":"Needle-in-a-haystack multi-query benchmark at 128K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","QA","Text"]},"mimo-niah-multi-256k":{"id":"mimo-niah-multi-256k","name":"NIAH-Multi 256K","topic":"Long-context QA","description":"Needle-in-a-haystack multi-query benchmark at 256K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","QA","Text"]},"mimo-niah-multi-32k":{"id":"mimo-niah-multi-32k","name":"NIAH-Multi 32K","topic":"Long-context QA","description":"Needle-in-a-haystack multi-query benchmark at 32K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","QA","Text"]},"mimo-niah-multi-64k":{"id":"mimo-niah-multi-64k","name":"NIAH-Multi 64K","topic":"Long-context QA","description":"Needle-in-a-haystack multi-query benchmark at 64K context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","QA","Text"]},"mindcube":{"id":"mindcube","name":"MindCube","topic":"Spatial navigation","description":"Spatial navigation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Navigation","Spatial","Vision"]},"mindcube-tiny":{"id":"mindcube-tiny","name":"MindCube-Tiny","topic":"Spatial reasoning","description":"Tiny evaluation subset of MindCube spatial mental-model benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"minerva-math":{"id":"minerva-math","name":"Minerva Math","topic":"University-level math","description":"Advanced quantitative reasoning set inspired by the Minerva benchmark for STEM problem solving.","url":"https://github.com/allenai/minerva-math","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/allenai/minerva-math","paper":"https://arxiv.org/abs/2206.14858","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"minerva-video":{"id":"minerva-video","name":"Minerva (Video)","topic":"Video reasoning","description":"Minerva video question-answering benchmark with complex multi-step reasoning over videos (not the Minerva math set).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Video","Reasoning"]},"minif2f-pass1":{"id":"minif2f-pass1","name":"MiniF2F pass@1","topic":"Math competition","description":"MiniF2F competition benchmark pass@1 accuracy.","url":"https://github.com/deepmind/mathematics_dataset","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"minif2f-pass32":{"id":"minif2f-pass32","name":"MiniF2F pass@32","topic":"Math competition","description":"MiniF2F competition benchmark pass@32 accuracy.","url":"https://github.com/deepmind/mathematics_dataset","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"minif2f-test":{"id":"minif2f-test","name":"MiniF2F (Test)","topic":"Math competition","description":"MiniF2F competition benchmark (test split).","url":"https://github.com/deepmind/mathematics_dataset","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"mirb":{"id":"mirb","name":"MIRB","topic":"Multi-image reasoning","description":"Multi-Image Relational Benchmark (average over perception, visual world knowledge, reasoning, multi-hop).","url":"https://github.com/ys-zong/MIRB","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2406.12742","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"mixcount":{"id":"mixcount","name":"MixCount","topic":"Visual counting","description":"MixCount test object counting, exact-match accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Counting"]},"mixeval":{"id":"mixeval","name":"MixEval","topic":"Multi-task reasoning","description":"Mixed-subject benchmark covering knowledge and reasoning tasks across domains.","url":"","stars":"","updated_at":"2025-07-29","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"mixeval-hard":{"id":"mixeval-hard","name":"MixEval Hard","topic":"Multi-task reasoning (hard)","description":"Hard subset of MixEval covering diverse reasoning tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"mle-bench":{"id":"mle-bench","name":"MLE-Bench","topic":"ML engineering","description":"OpenAI MLE-Bench: Kaggle-style machine-learning engineering competitions solved by agents.","url":"https://github.com/openai/mle-bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"mle-bench-30":{"id":"mle-bench-30","name":"MLE-bench-30","topic":"ML engineering","description":"30-competition subset of MLE-bench (Kaggle ML engineering tasks); average position score as used in the Gemini 3.6 Flash model card.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Research"]},"mle-bench-lite":{"id":"mle-bench-lite","name":"MLE-Bench Lite","topic":"ML engineering","description":"MLE-Bench Lite: 22-competition low-complexity subset of OpenAI MLE-Bench; metric is medal rate.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Coding","Text"]},"mlogiqa":{"id":"mlogiqa","name":"MLogiQA","topic":"Logical reasoning (multi-step)","description":"Multi-step logical reasoning QA benchmark.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"mls-bench-lite":{"id":"mls-bench-lite","name":"MLS-Bench-Lite","topic":"Agentic ML engineering","description":"Lite version of MLS-Bench, a long-horizon agentic machine-learning research/engineering benchmark with an official leaderboard.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"mlvu":{"id":"mlvu","name":"MLVU","topic":"Large video understanding","description":"MLVU: Large-scale multi-task benchmark for video understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"mm-browsecomp":{"id":"mm-browsecomp","name":"MM-BrowseComp","topic":"Multimodal browsing","description":"Multimodal browsing comprehension benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Web","Multimodal","Vision"]},"mm-clawbench":{"id":"mm-clawbench","name":"MM-ClawBench","topic":"Personal agent tasks","description":"MiniMax MM-ClawBench (MM Claw): end-to-end OpenClaw-style personal-agent benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"mm-ifeval":{"id":"mm-ifeval","name":"MM-IFEval","topic":"Multimodal instruction following","description":"Instruction-following benchmark assessing multimodal obedience to complex prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2502.04688","data":"https://huggingface.co/datasets/lscpku/MMIFEval","modality":"multimodal","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Multimodal","Vision","Text"]},"mm-mt-bench":{"id":"mm-mt-bench","name":"MM-MT-Bench","topic":"Multimodal instruction following","description":"Multi-turn multimodal instruction following benchmark evaluating dialogue quality and helpfulness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Instruction Following","Multimodal","Vision"]},"mmar":{"id":"mmar","name":"MMAR","topic":"Audio reasoning","description":"MMAR: deep reasoning over speech, sound, music and their mix (%).","url":"https://github.com/ddlBoJack/MMAR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2505.13032","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Reasoning"]},"mmau-pro":{"id":"mmau-pro","name":"MMAU-Pro","topic":"Audio reasoning","description":"MMAU-Pro: challenging audio understanding and reasoning benchmark (speech, sound, music).","url":"https://arxiv.org/abs/2508.13992","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Reasoning"]},"mmbench":{"id":"mmbench","name":"MMBench (EN)","topic":"Multimodal understanding (English)","description":"MMBench v1.0 English test split.","url":"https://mmbench.opencompass.org.cn/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2307.06281","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmbench-cn":{"id":"mmbench-cn","name":"MMBench (CN)","topic":"Multimodal understanding (Chinese)","description":"MMBench v1.0 Chinese test split.","url":"https://mmbench.opencompass.org.cn/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2307.06281","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Multimodal","Vision"]},"mmbench-gui-l2":{"id":"mmbench-gui-l2","name":"MMBench-GUI L2","topic":"GUI grounding","description":"MMBench-GUI level-2 element grounding across Windows, macOS, Linux, iOS, Android and Web.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2507.19478","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"mmbench-v1.1-cn":{"id":"mmbench-v1.1-cn","name":"MMBench v1.1 (CN)","topic":"Multimodal understanding (Chinese)","description":"MMBench v1.1 Chinese subset for evaluating multimodal LLMs.","url":"https://mmbench.opencompass.org.cn/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2307.06281","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Multimodal","Vision"]},"mmbench-v1.1-en":{"id":"mmbench-v1.1-en","name":"MMBench v1.1 (EN)","topic":"Multimodal understanding (English)","description":"MMBench v1.1 English subset for evaluating multimodal LLMs.","url":"https://mmbench.opencompass.org.cn/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2307.06281","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmbench-v1.1-en-dev":{"id":"mmbench-v1.1-en-dev","name":"MMBench v1.1 (EN dev)","topic":"General VQA","description":"English dev split of MMBench v1.1 measuring multimodal question answering.","url":"https://mmbench.opencompass.org.cn/home","stars":"","updated_at":"","leaderboard":"https://mmbench.opencompass.org.cn/leaderboard","github":"https://github.com/open-compass/MMBench","paper":"https://arxiv.org/abs/2307.06281","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"mmbench-video":{"id":"mmbench-video","name":"MMBench-Video","topic":"Video QA","description":"Long-form multi-shot video understanding with GPT-4-judged free-form answers; overall mean score on a 0-3 scale.","url":"https://github.com/open-compass/MMBench-Video","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2406.14515","modality":"video","metric":"score","higher_is_better":true,"classification":["Multimodal","Video","Vision"]},"mmc-benchmark":{"id":"mmc-benchmark","name":"MMC-Benchmark","topic":"Chart understanding","description":"MMC multimodal chart understanding benchmark.","url":"https://github.com/FuxiaoLiu/MMC","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2311.10774","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmdu":{"id":"mmdu","name":"MMDU","topic":"Multi-turn multi-image dialog","description":"Multi-turn, multi-image dialog understanding benchmark with GPT-4o-judged scores.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/Liuziyu77/MMDU","paper":"https://arxiv.org/abs/2406.11833","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision","Chat"]},"mme-cc":{"id":"mme-cc","name":"MME-CC","topic":"Multimodal evaluation","description":"MME-CC multimodal evaluation suite.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mme-elo":{"id":"mme-elo","name":"MME Elo","topic":"Multimodal perception","description":"Elo-style scoring for the MME multimodal evaluation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"points","higher_is_better":true,"classification":["Perception","Multimodal","Vision"]},"mme-realworld-cn":{"id":"mme-realworld-cn","name":"MME-RealWorld (cn)","topic":"Real-world perception (CN)","description":"MME-RealWorld Chinese split.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Perception","Chinese","Multimodal","Vision"]},"mme-realworld-en":{"id":"mme-realworld-en","name":"MME-RealWorld (en)","topic":"Real-world perception (EN)","description":"MME-RealWorld English split.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Perception","Multimodal","Vision"]},"mmhal-bench":{"id":"mmhal-bench","name":"MMHal-Bench (Score)","topic":"Multimodal hallucination","description":"MMHal-Bench GPT-judged response quality score (0-6) for hallucination in vision-language models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2309.14525","modality":"visual","metric":"score","higher_is_better":true,"classification":["Hallucination","Multimodal","Vision"]},"mmhal-bench-hallucination-rate":{"id":"mmhal-bench-hallucination-rate","name":"MMHal-Bench (Hallucination Rate)","topic":"Multimodal hallucination","description":"MMHal-Bench hallucination rate of vision-language model responses; lower is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2309.14525","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Hallucination","Multimodal","Vision"]},"mmiu":{"id":"mmiu","name":"MMIU","topic":"Multi-image understanding","description":"Multi-image understanding benchmark evaluating cross-image reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"mmlb-niah-128k":{"id":"mmlb-niah-128k","name":"MMLB-NIAH (128k)","topic":"Multimodal long-context","description":"MMLB-NIAH 128k long-context multimodal benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Long Context","Multimodal","Vision"]},"mmlb-vrag-128k":{"id":"mmlb-vrag-128k","name":"MMLB-VRAG (128k)","topic":"Multimodal long-context","description":"MMLB-VRAG 128k long-context multimodal benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Long Context","Multimodal","Vision"]},"mmlongbench-128k":{"id":"mmlongbench-128k","name":"MMLongBench-128K","topic":"Long-context multimodal","description":"128K-context variant of MMLongBench evaluating multimodal long-context understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Long Context","Multimodal","Vision"]},"mmlongbench-doc":{"id":"mmlongbench-doc","name":"MMLongBench-Doc","topic":"Long-context multimodal documents","description":"Evaluates long-context document understanding with mixed text, tables, and figures across multiple pages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Long Context","Multimodal","Vision"]},"mmlu":{"id":"mmlu","name":"MMLU","topic":"Multi-domain knowledge","description":"57 tasks spanning STEM, humanities, social sciences; broad knowledge and reasoning.","url":"https://github.com/hendrycks/test","stars":"1488","updated_at":"2025-08-30","leaderboard":"https://www.kaggle.com/benchmarks/open-benchmarks/mmlu","github":"https://github.com/hendrycks/test","paper":"https://arxiv.org/abs/2009.03300","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","STEM","Science","Text"]},"mmlu-arabic":{"id":"mmlu-arabic","name":"MMLU Arabic","topic":"Arabic knowledge and reasoning","description":"Arabic-language variant of MMLU evaluating knowledge and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","QA","Text"]},"mmlu-cf":{"id":"mmlu-cf","name":"MMLU-CF","topic":"Multi-domain knowledge (few-shot)","description":"MMLU common-format few-shot evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-cloze":{"id":"mmlu-cloze","name":"MMLU (cloze)","topic":"Multi-domain knowledge (cloze)","description":"Cloze-form MMLU evaluation variant.","url":"https://github.com/hendrycks/test","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/hendrycks/test","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-fulltext":{"id":"mmlu-fulltext","name":"Full Text MMLU","topic":"Multi-domain knowledge (long-form)","description":"Full-context MMLU variant evaluating reasoning over long passages.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"norm_log_acc","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Long Context","Text"]},"mmlu-med":{"id":"mmlu-med","name":"MMLU Med","topic":"Medical knowledge","description":"Medical subset of MMLU (clinical knowledge, anatomy, college medicine, medical genetics, professional medicine, college biology).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"mmlu-pro":{"id":"mmlu-pro","name":"MMLU-Pro","topic":"Multi-domain knowledge","description":"Harder successor to MMLU with more challenging questions.","url":"https://github.com/TIGER-AI-Lab/MMLU-Pro","stars":"286","updated_at":"2025-09-08","leaderboard":"https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro","github":"https://github.com/TIGER-AI-Lab/MMLU-Pro","paper":"https://arxiv.org/abs/2406.01574","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-pro-cf":{"id":"mmlu-pro-cf","name":"MMLU Pro CF","topic":"Multi-domain knowledge (few-shot)","description":"MMLU-Pro common-format (CF) few-shot evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-pro-mcf":{"id":"mmlu-pro-mcf","name":"MMLU Pro MCF","topic":"Multi-domain knowledge (few-shot)","description":"MMLU-Pro common format (MCF) few-shot evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-prox":{"id":"mmlu-prox","name":"MMLU-ProX","topic":"Multi-domain knowledge","description":"Cross-lingual and robust variant of MMLU-Pro.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-prox-ja":{"id":"mmlu-prox-ja","name":"MMLU-ProX (Japanese)","topic":"Multi-domain knowledge (Japanese)","description":"Japanese subset of MMLU-ProX (JMMLU-ProX).","url":"https://mmluprox.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text","Japanese"]},"mmlu-redux":{"id":"mmlu-redux","name":"MMLU-Redux","topic":"Multi-domain knowledge","description":"Updated MMLU-style evaluation with revised questions and scoring.","url":"","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/allenai/ZeroEval","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmlu-stem":{"id":"mmlu-stem","name":"MMLU-STEM","topic":"STEM knowledge","description":"STEM subset of MMLU.","url":"https://github.com/hendrycks/test","stars":"1488","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmmb":{"id":"mmmb","name":"MMMB","topic":"Multilingual MMBench","description":"Multilingual Multimodal Benchmark (MMMB) average score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multilingual","Multimodal","Vision"]},"mmmlu":{"id":"mmmlu","name":"MMMLU","topic":"Multi-domain knowledge (multilingual)","description":"Massively multilingual MMLU-style evaluation across many languages.","url":"https://huggingface.co/LiquidAI/LFM2-350M","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmmlu-es":{"id":"mmmlu-es","name":"MMMLU (ES)","topic":"Multilingual knowledge","description":"Spanish MMMLU benchmark","url":"","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Text"]},"mmmlu-german":{"id":"mmmlu-german","name":"MMMLU-German","topic":"Multilingual knowledge (German)","description":"German subset of multilingual MMLU-style evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"mmmu":{"id":"mmmu","name":"MMMU","topic":"Multimodal understanding","description":"Multi-discipline multimodal understanding benchmark.","url":"https://mmmu-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmmu-med":{"id":"mmmu-med","name":"MMMU-Med","topic":"Medical QA","description":"Medical/health subset of MMMU (validation).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"mmmu-pro":{"id":"mmmu-pro","name":"MMMU PRO","topic":"Multimodal understanding (hard)","description":"Professional/advanced subset of MMMU for multimodal reasoning.","url":"https://mmmu-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"mmmu-pro-vision":{"id":"mmmu-pro-vision","name":"MMMU-Pro (vision)","topic":"Multimodal understanding (vision)","description":"MMMU-Pro vision-only setting.","url":"https://mmmu-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmmu-pro-with-tools":{"id":"mmmu-pro-with-tools","name":"MMMU Pro (with tools)","topic":"Multimodal understanding (with tools)","description":"MMMU-Pro benchmark evaluated with tool access.","url":"https://mmmu-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Tools","Vision"]},"mmmu-test":{"id":"mmmu-test","name":"MMMU (test)","topic":"Multimodal understanding","description":"MMMU test-set accuracy (multi-discipline college-level multimodal questions).","url":"https://mmmu-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2311.16502","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"mmsearch":{"id":"mmsearch","name":"MMSearch","topic":"Multimodal search","description":"Benchmark for multimodal search engines / LMMs answering queries via image+web search.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Web","Multimodal","Vision"]},"mmsearch-plus":{"id":"mmsearch-plus","name":"MMSearch-Plus","topic":"Multimodal search","description":"Harder MMSearch variant requiring fine-grained visual reasoning plus multi-step search.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Web","Multimodal","Vision"]},"mmsi-bench":{"id":"mmsi-bench","name":"MMSI-Bench","topic":"Spatial reasoning","description":"Multi-image spatial intelligence benchmark (standard, non-circular evaluation).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"mmsibench-circular":{"id":"mmsibench-circular","name":"MMSIBench (circular)","topic":"Spatial understanding","description":"MMSIBench circular subset for spatial reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"mmstar":{"id":"mmstar","name":"MMStar","topic":"Multimodal reasoning","description":"Broad evaluation of multimodal LLMs across diverse tasks.","url":"https://mmstar-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/MMStar-Benchmark/MMStar","paper":"https://arxiv.org/abs/2403.20330","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"mmt-bench":{"id":"mmt-bench","name":"MMT-Bench","topic":"Multimodal multitask understanding","description":"Large-scale multimodal multitask benchmark covering diverse meta-tasks for LVLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.16006","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmvet":{"id":"mmvet","name":"MMVet","topic":"Multimodal evaluation","description":"Comprehensive evaluation suite for assessing multimodal LLM capabilities.","url":"https://mm-vet.github.io/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/yuweihao/MM-Vet","paper":"https://arxiv.org/abs/2308.02490","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmvet-v2":{"id":"mmvet-v2","name":"MM-Vet v2","topic":"Multimodal evaluation","description":"MM-Vet v2: integrated capabilities incl. image-text sequence understanding; GPT-4 graded.","url":"https://github.com/yuweihao/MM-Vet","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2408.00765","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"mmvp":{"id":"mmvp","name":"MMVP","topic":"Multimodal video perception","description":"Benchmark for multimodal video understanding and perception.","url":"https://mmvp-benchmark.github.io/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/Metaverse-AI-Lab-THU/MMVP-Dataset","paper":"https://arxiv.org/abs/2308.16154","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Video","Vision"]},"mmvu":{"id":"mmvu","name":"MMVU","topic":"Video understanding","description":"Multimodal video understanding benchmark (MMVU).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"mobile-actions":{"id":"mobile-actions","name":"Mobile Actions","topic":"Tool use","description":"Google Mobile Actions dataset: function calling for on-device Android actions (FunctionGemma eval).","url":"https://huggingface.co/datasets/google/mobile-actions","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tool Use","Mobile"]},"mobileworld":{"id":"mobileworld","name":"MobileWorld","topic":"Mobile GUI agents","description":"Benchmark of agents completing tasks in mobile (Android) app environments.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"mol-instructions":{"id":"mol-instructions","name":"Mol-Instructions","topic":"Bio-molecular instruction following","description":"Instruction-following benchmark for bio-molecular understanding and generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chemistry","Science","STEM","Vision"]},"molbench-bind":{"id":"molbench-bind","name":"MolBench-bind","topic":"Molecular science agents","description":"MolBench binding-affinity subset: scientific agentic tasks on molecular binding prediction.","url":"https://huggingface.co/InternScience/Agents-A1","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Science","Text"]},"molecular-iq":{"id":"molecular-iq","name":"MolecularIQ","topic":"Molecular structure reasoning","description":"Benchmark of molecular structure reasoning for chemistry-capable LLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chemistry","Science","Text"]},"mortgagetax":{"id":"mortgagetax","name":"MortgageTax","topic":"Finance QA","description":"Question answering benchmark focused on mortgage and tax regulations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Knowledge","Text"]},"motionbench":{"id":"motionbench","name":"MotionBench","topic":"Video motion understanding","description":"Video motion and temporal reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"mpdocbench-truncated-table-teds":{"id":"mpdocbench-truncated-table-teds","name":"MPDocBench-Parse Truncated Table TEDS","topic":"Cross-page tables","description":"MPDocBench-Parse: TEDS on truncated (page-split) tables.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"mrcr-128k":{"id":"mrcr-128k","name":"OpenAI-MRCR (128k)","topic":"Long-context reasoning","description":"OpenAI Multi-Round Chain Reasoning benchmark with 128k context window.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-128k-2n":{"id":"mrcr-128k-2n","name":"MRCR 128K-2N","topic":"Long-context reasoning","description":"Multi-Round Coreference Resolution benchmark at 128k context with 2 needles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-128k-4n":{"id":"mrcr-128k-4n","name":"MRCR 128K-4N","topic":"Long-context reasoning","description":"Multi-Round Coreference Resolution benchmark at 128k context with 4 needles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-128k-8n":{"id":"mrcr-128k-8n","name":"MRCR 128K-8N","topic":"Long-context reasoning","description":"Multi-Round Coreference Resolution benchmark at 128k context with 8 needles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-16k-8n":{"id":"mrcr-16k-8n","name":"MRCR 16K-8N","topic":"Long-context reasoning","description":"OpenAI MRCR v2 8-needle, 8K-16K context bucket.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-1m":{"id":"mrcr-1m","name":"OpenAI-MRCR (1M)","topic":"Long-context reasoning","description":"OpenAI Multi-Round Chain Reasoning benchmark with 1M context window.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-1m-8n":{"id":"mrcr-1m-8n","name":"MRCR 1M-8N","topic":"Long-context reasoning","description":"OpenAI MRCR v2, 8 needles, 512K-1M context bucket.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-256k-8n":{"id":"mrcr-256k-8n","name":"MRCR 256K-8N","topic":"Long-context reasoning","description":"OpenAI MRCR v2 8-needle, 128K-256K context bucket.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-32k-8n":{"id":"mrcr-32k-8n","name":"MRCR 32K-8N","topic":"Long-context reasoning","description":"OpenAI MRCR v2 8-needle, 16K-32K context bucket.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-512k-8n":{"id":"mrcr-512k-8n","name":"MRCR 512K-8N","topic":"Long-context reasoning","description":"OpenAI MRCR v2, 8 needles, 256K-512K context bucket.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-64k-2n":{"id":"mrcr-64k-2n","name":"MRCR 64K-2N","topic":"Long-context reasoning","description":"Multi-Round Coreference Resolution benchmark at 64k context with 2 needles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-64k-4n":{"id":"mrcr-64k-4n","name":"MRCR 64K-4N","topic":"Long-context reasoning","description":"Multi-Round Coreference Resolution benchmark at 64k context with 4 needles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-64k-8n":{"id":"mrcr-64k-8n","name":"MRCR 64K-8N","topic":"Long-context reasoning","description":"Multi-Round Coreference Resolution benchmark at 64k context with 8 needles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-8k-8n":{"id":"mrcr-8k-8n","name":"MRCR 8K-8N","topic":"Long-context reasoning","description":"OpenAI MRCR v2 8-needle, 4K-8K context bucket.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mrcr-v2":{"id":"mrcr-v2","name":"MRCR v2","topic":"Multimodal reasoning","description":"Multi-round multimodal chain-of-reasoning evaluation (v2).","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"ms-cxr-t":{"id":"ms-cxr-t","name":"MS-CXR-T","topic":"Medical imaging","description":"Temporal chest X-ray disease progression classification, macro accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"msearth-mcq":{"id":"msearth-mcq","name":"MSEarth-MCQ","topic":"Earth science","description":"Earth science multiple-choice question benchmark for scientific AI models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Earth Science","Science","STEM","Vision"]},"msvd-qa":{"id":"msvd-qa","name":"MSVD-QA","topic":"Video QA (open-ended)","description":"Open-ended QA on MSVD video clips; GPT-judged accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1702.00002","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"mt-bench":{"id":"mt-bench","name":"MT-Bench","topic":"Chat ability","description":"Multi-turn chat evaluation via GPT-4 grading.","url":"https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge","stars":"39074","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2306.05685v4","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"mt-bench-ja":{"id":"mt-bench-ja","name":"Japanese MT-Bench","topic":"Multi-turn chat (Japanese)","description":"Japanese adaptation of MT-Bench multi-turn conversational benchmark, LLM-as-judge score on a 1-10 scale.","url":"https://github.com/Stability-AI/FastChat/tree/jp-stable/fastchat/llm_judge","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"score","higher_is_better":true,"classification":["Chat","Japanese","Text"]},"mtalk-bench":{"id":"mtalk-bench","name":"MTalk-Bench","topic":"Spoken multi-turn dialogue","description":"MTalk-Bench: multi-turn speech-to-speech dialogue (semantic, paralinguistic, ambient) evaluation (%).","url":"https://arxiv.org/abs/2508.18240","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Chat"]},"mteb-code-v1":{"id":"mteb-code-v1","name":"MTEB (Code, v1)","topic":"Code embeddings","description":"Massive Text Embedding Benchmark, code tasks v1.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"mteb-english-v2":{"id":"mteb-english-v2","name":"MTEB (English, v2)","topic":"Text embeddings (English)","description":"Massive Text Embedding Benchmark, English v2.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"mteb-multilingual-v2":{"id":"mteb-multilingual-v2","name":"MTEB Multilingual v2","topic":"Text embeddings (multilingual)","description":"Massive Text Embedding Benchmark, multilingual v2.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"mtob-full-book":{"id":"mtob-full-book","name":"MTOB (full book)","topic":"Long-form reasoning","description":"Long-context book understanding benchmark (full-book setting).","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mtob-half-book":{"id":"mtob-half-book","name":"MTOB (half book)","topic":"Long-form reasoning","description":"Long-context book understanding benchmark (half-book setting).","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"mtvqa":{"id":"mtvqa","name":"MTVQA","topic":"Multilingual text-centric VQA","description":"Multilingual text-centric visual question answering across 9 languages (average accuracy).","url":"https://bytedance.github.io/MTVQA/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2405.11985","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multilingual","OCR","Multimodal","Vision"]},"muirbench":{"id":"muirbench","name":"MUIRBENCH","topic":"Multimodal robustness","description":"Evaluates multimodal understanding robustness and reliability.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision","Robustness"]},"multi-if":{"id":"multi-if","name":"Multi-IF","topic":"Instruction following (multi-task)","description":"Composite instruction-following evaluation across multiple tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"multi-ifeval":{"id":"multi-ifeval","name":"Multi-IFEval","topic":"Instruction following (multi-task)","description":"Multi-task variant of instruction-following evaluation.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"multi-swe-bench":{"id":"multi-swe-bench","name":"Multi-SWE-Bench","topic":"Code repair (multi-repo)","description":"Multi-repository SWE-Bench variant.","url":"https://multi-swe-bench.github.io","stars":"246","updated_at":"2025-09-05","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"multi-swe-bench-flash":{"id":"multi-swe-bench-flash","name":"Multi-SWE-Bench-Flash","topic":"Code repair (multi-repo)","description":"Flash subset of Multi-SWE-Bench.","url":"https://multi-swe-bench.github.io","stars":"246","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"multi-swe-bench-mini":{"id":"multi-swe-bench-mini","name":"Multi-SWE-Bench-Mini","topic":"Code repair (multi-repo)","description":"Mini subset of Multi-SWE-Bench.","url":"https://multi-swe-bench.github.io","stars":"246","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"multichallenge":{"id":"multichallenge","name":"MultiChallenge","topic":"Instruction following","description":"Multi-domain instruction-following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"multiimg-qa-avg":{"id":"multiimg-qa-avg","name":"Multi-Image QA Average","topic":"Multi-image QA (aggregate)","description":"Aggregate score over multi-image visual question answering tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"multilingual-jailbreak":{"id":"multilingual-jailbreak","name":"Multilingual","topic":"Safety / jailbreak","description":"Jailbreak resistance across multilingual adversarial prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Multilingual","Text"]},"multilingual-mmbench":{"id":"multilingual-mmbench","name":"Multilingual MMBench","topic":"Multilingual vision benchmark","description":"Multilingual MMBench average score across languages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multilingual","Multimodal","Vision"]},"multilingual-mmlu":{"id":"multilingual-mmlu","name":"Multilingual MMLU","topic":"Multi-domain knowledge (multilingual)","description":"Multilingual variant of MMLU across many languages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"multiloko":{"id":"multiloko","name":"MultiLoKo","topic":"Multilingual knowledge","description":"Multilingual local-knowledge QA benchmark covering 31 languages with locally sourced questions.","url":"https://github.com/facebookresearch/multiloko","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual"]},"multipl-e":{"id":"multipl-e","name":"MultiPL-E","topic":"Code generation (multilingual)","description":"Multilingual code generation and execution benchmark across many programming languages.","url":"https://github.com/nuprl/MultiPL-E","stars":"269","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"multipl-e-humaneval":{"id":"multipl-e-humaneval","name":"MultiPL-E HumanEval","topic":"Code generation (multilingual)","description":"MultiPL-E variant of HumanEval tasks.","url":"https://github.com/nuprl/MultiPL-E","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"multipl-e-mbpp":{"id":"multipl-e-mbpp","name":"MultiPL-E MBPP","topic":"Code generation (multilingual)","description":"MultiPL-E variant of MBPP tasks.","url":"https://github.com/nuprl/MultiPL-E","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"musr":{"id":"musr","name":"MuSR","topic":"Reasoning","description":"Multistep Soft Reasoning.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2310.16049","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"mvbench":{"id":"mvbench","name":"MVBench","topic":"Video QA","description":"Multi-view or multi-video QA benchmark (MVBench).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"nanochat-autoresearch":{"id":"nanochat-autoresearch","name":"NanoChat AutoResearch","topic":"AI research automation","description":"Agents improve NanoChat language-model quality under a fixed training budget; validation bits-per-byte (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"bpb","higher_is_better":false,"classification":["Agents","Code","Research"]},"nanogpt":{"id":"nanogpt","name":"NanoGPT","topic":"AI research automation","description":"OpenAI internal eval (GPT-5.6 announcement): improving a NanoGPT training recipe (speed-up score).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Research","Text"]},"nanogpt-speedrun":{"id":"nanogpt-speedrun","name":"NanoGPT SpeedRun","topic":"AI research automation","description":"Agents reduce NanoGPT training time to a target validation loss; training time in seconds (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"seconds","higher_is_better":false,"classification":["Agents","Code","Research"]},"nanonets-kie":{"id":"nanonets-kie","name":"Nanonets KIE","topic":"Key information extraction","description":"Nanonets key information extraction benchmark (F1).","url":"https://github.com/NanoNets/docext","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"natural2code":{"id":"natural2code","name":"Natural2Code","topic":"Code generation","description":"Natural language to code benchmark for instruction-following synthesis.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"naturalcodebench":{"id":"naturalcodebench","name":"NaturalCodeBench","topic":"Code generation","description":"NaturalCodeBench (NCB): real-world coding prompts in Python and Java, English and Chinese.","url":"https://github.com/THUDM/NaturalCodeBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2405.04520","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"naturalquestions":{"id":"naturalquestions","name":"NaturalQuestions","topic":"Open-domain QA","description":"Google NQ; real user questions with long/short answers.","url":"https://ai.google.com/research/NaturalQuestions","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"navsim":{"id":"navsim","name":"NAVSIM (PDMS)","topic":"Driving planning","description":"NAVSIM v1 non-reactive simulation benchmark for end-to-end driving planners; score = PDM Score (PDMS, 0-100).","url":"https://github.com/autonomousvision/navsim","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Autonomous Driving","Vision"]},"new-yorker-caption-matching":{"id":"new-yorker-caption-matching","name":"New Yorker caption matching","topic":"Humor understanding","description":"Pick the caption written for a described New Yorker cartoon from five; accuracy (Jev Decision Index suite, posed as typed decisions).","url":"https://huggingface.co/datasets/jmhessel/newyorker_caption_contest","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Humor","Text"]},"nextbase":{"id":"nextbase","name":"NextBase","topic":"Genomics","description":"Next-base DNA sequence prediction (per-base accuracy).","url":"https://huggingface.co/HuggingFaceBio/Carbon-X","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biology","STEM","Text"]},"nextqa":{"id":"nextqa","name":"NExT-QA","topic":"Video understanding","description":"Video question answering benchmark on causal and temporal action reasoning (multiple-choice).","url":"https://github.com/doc-doc/NExT-QA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":""},"nexus":{"id":"nexus","name":"Nexus (0-shot)","topic":"Tool use","description":"Nexus tool-use benchmark, zero-shot setting.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tools","Text"]},"nih":{"id":"nih","name":"Needle In A Haystack","topic":"Long-context retrieval","description":"Needle In A Haystack test for locating hidden facts in long contexts.","url":"https://github.com/gkamradt/LLMTest_NeedleInAHaystack","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/gkamradt/LLMTest_NeedleInAHaystack","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"nl2repo":{"id":"nl2repo","name":"NL2Repo","topic":"Software engineering","description":"Long-horizon repository generation: agents build a complete Python repository from a natural-language requirements document; scored by the repo's test suite.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"nli4ct":{"id":"nli4ct","name":"NLI4CT","topic":"Clinical NLI","description":"NLI over clinical trial reports (SemEval NLI4CT); macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://sites.google.com/view/nli4ct/","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["NLI","Medical","Text"]},"nocaps":{"id":"nocaps","name":"NoCaps","topic":"Image captioning","description":"Novel object captioning benchmark (CIDEr on val).","url":"https://nocaps.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1812.08658","modality":"visual","metric":"cider","higher_is_better":true,"classification":["Captioning","Multimodal","Vision"]},"noharm":{"id":"noharm","name":"NOHARM","topic":"Clinical safety","description":"Benchmark of clinical-harm avoidance in medical recommendations (Stanford/ARiSE NOHARM).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Safety","Text"]},"nolima-128k":{"id":"nolima-128k","name":"NoLiMa 128K","topic":"Long-context eval","description":"NoLiMa (No Literal Match) long-context benchmark at 128k context window.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"nolima-32k":{"id":"nolima-32k","name":"NoLiMa 32K","topic":"Long-context eval","description":"NoLiMa (No Literal Match) long-context benchmark at 32k context window.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"nolima-64k":{"id":"nolima-64k","name":"NoLiMa 64K","topic":"Long-context eval","description":"NoLiMa (No Literal Match) long-context benchmark at 64k context window.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"nova-63":{"id":"nova-63","name":"NOVA-63","topic":"Multilingual evaluation","description":"Multilingual evaluation benchmark covering 63 languages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"nuscenes":{"id":"nuscenes","name":"NuScenes","topic":"3D scene understanding","description":"3D scene understanding and perception benchmark for autonomous driving.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["3D","Autonomous Driving","Vision"]},"objectron":{"id":"objectron","name":"Objectron","topic":"Object detection","description":"Objectron benchmark for 3D object detection in video captures.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Detection","Multimodal","Vision"]},"obqa":{"id":"obqa","name":"OBQA","topic":"Open book QA","description":"OpenBookQA science question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"ocnli":{"id":"ocnli","name":"OCNLI","topic":"Natural language inference (Chinese)","description":"Original Chinese Natural Language Inference benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chinese","NLI","Reasoning","Text"]},"ocrbench":{"id":"ocrbench","name":"OCRBench V2","topic":"OCR (vision text extraction)","description":"OCRBench v2 evaluating text extraction from images and documents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","Vision"]},"ocrbench-elo":{"id":"ocrbench-elo","name":"OCRBench-ELO","topic":"OCR (ELO ranking)","description":"OCR benchmark using ELO rating system to rank model performance on text extraction tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"elo","higher_is_better":true,"classification":["Document","Multimodal","Vision"]},"ocrbench-kie":{"id":"ocrbench-kie","name":"OCRBench (KIE)","topic":"Key information extraction","description":"Key information extraction subset of OCRBench (v1).","url":"https://github.com/Yuliang-Liu/MultimodalOCR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"ocrbenchv2-cn":{"id":"ocrbenchv2-cn","name":"OCRBenchV2 (CN)","topic":"OCR (Chinese)","description":"OCRBenchV2 Chinese subset assessing OCR performance on Chinese-language documents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Document","Multimodal","Vision"]},"ocrbenchv2-en":{"id":"ocrbenchv2-en","name":"OCRBenchV2 (EN)","topic":"OCR (English)","description":"OCRBenchV2 English subset evaluating OCR accuracy on English documents and layouts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","Vision"]},"ocrbenchv2-kie":{"id":"ocrbenchv2-kie","name":"OCRBenchV2 KIE","topic":"Key information extraction","description":"Key information extraction subset of OCRBench v2 as reported without a language split (OvisOCR3 release).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"ocrbenchv2-kie-en":{"id":"ocrbenchv2-kie-en","name":"OCRBenchV2 KIE (EN)","topic":"Key information extraction","description":"English key information extraction subset of OCRBench v2.","url":"https://github.com/Yuliang-Liu/MultimodalOCR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"ocrbenchv2-kie-zh":{"id":"ocrbenchv2-kie-zh","name":"OCRBenchV2 KIE (ZH)","topic":"Key information extraction (Chinese)","description":"Chinese key information extraction subset of OCRBench v2.","url":"https://github.com/Yuliang-Liu/MultimodalOCR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chinese","Document","Multimodal","OCR","Vision"]},"ocrreasoning":{"id":"ocrreasoning","name":"OCRReasoning","topic":"OCR reasoning","description":"OCR reasoning benchmark combining text extraction with multi-step reasoning over documents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","Reasoning","Vision"]},"ocrvqa":{"id":"ocrvqa","name":"OCR-VQA","topic":"Text-based VQA","description":"Visual question answering by reading text in book cover images.","url":"https://ocr-vqa.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","OCR","QA","Vision"]},"octocodingbench":{"id":"octocodingbench","name":"OctoCodingBench","topic":"Code generation","description":"Coding benchmark across multi-language programming tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"ocw-courses":{"id":"ocw-courses","name":"OCW Courses","topic":"STEM math reasoning","description":"MIT OpenCourseWare undergraduate STEM problems (from Minerva).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2206.14858","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Science","Text"]},"odinw-13":{"id":"odinw-13","name":"ODinW-13","topic":"Object detection (in the wild)","description":"Object Detection in the Wild benchmark covering 13 real-world domains.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Detection","Multimodal","Vision"]},"odvbench":{"id":"odvbench","name":"ODVBench","topic":"Streaming video understanding","description":"Online/streaming video understanding benchmark (accuracy).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","QA","Vision"]},"odyssey-math":{"id":"odyssey-math","name":"Odyssey Math","topic":"Math reasoning","description":"Odyssey multi-step math benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"officeqa":{"id":"officeqa","name":"OfficeQA","topic":"Enterprise document QA","description":"Grounded question answering over office and enterprise documents, as reported by OpenAI for GPT-5.4.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","QA","Text"]},"officeqa-pro":{"id":"officeqa-pro","name":"OfficeQA Pro","topic":"Enterprise document QA","description":"Harder OfficeQA variant. Models must use files, tools and context together instead of answering from memory.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","QA","Text"]},"officeqa-pro-v2":{"id":"officeqa-pro-v2","name":"OfficeQA Pro v2","topic":"Enterprise document QA","description":"OfficeQA Pro v2: grounded QA over office/enterprise documents requiring files, tools and context.","url":"https://www.kimi.ai/blog/kimi-k3-1-pro","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","QA","Text"]},"oibench-en":{"id":"oibench-en","name":"OIBench EN","topic":"Code generation","description":"English subset of OIBench for code generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"ojbench":{"id":"ojbench","name":"OJBench","topic":"Code generation (online judge)","description":"Programming problems evaluated via online judge-style execution.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"okvqa":{"id":"okvqa","name":"OK-VQA","topic":"Knowledge-based VQA","description":"Visual question answering requiring outside knowledge.","url":"https://okvqa.allenai.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1906.00067","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"olmocr-bench":{"id":"olmocr-bench","name":"olmOCR-Bench","topic":"Document OCR","description":"olmOCR benchmark assessing OCR fidelity and structured extraction on complex document pages.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","OCR","Vision"]},"olympiadbench":{"id":"olympiadbench","name":"OlympiadBench","topic":"Math (olympiad)","description":"Advanced mathematics olympiad-style problem benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"olympic-arena":{"id":"olympic-arena","name":"OlympicArena","topic":"Math (competition)","description":"Olympiad-style mathematics reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"omega":{"id":"omega","name":"OMEGA","topic":"Math (advanced)","description":"OMEGA olympiad-grade mathematics reasoning benchmark.","url":"","stars":"","updated_at":"2025-11-20","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"omni-math":{"id":"omni-math","name":"Omni-MATH","topic":"Math reasoning","description":"Omni-MATH benchmark covering diverse math reasoning tasks across difficulty levels.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"omni-math-hard":{"id":"omni-math-hard","name":"Omni-MATH-HARD","topic":"Math","description":"Challenging math benchmark (Omni-MATH-HARD).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"omni3d":{"id":"omni3d","name":"Omni3D","topic":"3D detection","description":"Omni3D monocular 3D object detection benchmark across indoor/outdoor datasets (AP3D).","url":"https://github.com/facebookresearch/omni3d","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["3D","Detection","Vision"]},"omnidocbench":{"id":"omnidocbench","name":"OmniDocBench","topic":"Document understanding","description":"Document understanding benchmark covering multi-page layouts, tables, and charts for robust question answering. Overall score, higher is better (v1.0 edit-distance results are in omnidocbench-edit).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","QA","Vision"]},"omnidocbench-1.5":{"id":"omnidocbench-1.5","name":"OmniDocBench 1.5","topic":"OCR","description":"Document understanding benchmark v1.5 with OCR evaluation. Overall Edit Distance metric, lower is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"edit_distance","higher_is_better":false,"classification":["Document","Multimodal","OCR","Vision"]},"omnidocbench-cn":{"id":"omnidocbench-cn","name":"OmniDocBench-CN","topic":"Document understanding (Chinese)","description":"Chinese subset of OmniDocBench focusing on OCR-grounded document comprehension and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Chinese","Document","Multimodal","QA","Vision"]},"omnidocbench-edit":{"id":"omnidocbench-edit","name":"OmniDocBench (Edit Distance)","topic":"Document understanding","description":"OmniDocBench v1.0 overall normalized edit distance (x100), lower is better.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":false,"classification":["Document","Multimodal","QA","Vision"]},"omnidocbench-v1.6":{"id":"omnidocbench-v1.6","name":"OmniDocBench v1.6","topic":"Document parsing","description":"OmniDocBench v1.6 end-to-end document parsing; Overall score (higher is better) combining text edit, formula CDM and table TEDS.","url":"https://github.com/opendatalab/OmniDocBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"omnigaia":{"id":"omnigaia","name":"OmniGAIA","topic":"Multimodal agent","description":"OmniGAIA: GAIA-style omni-modal (image/video/audio) agentic perception and reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","Multimodal","Reasoning"]},"omnimedvqa":{"id":"omnimedvqa","name":"OmniMedVQA","topic":"Medical QA","description":"Large-scale medical VQA across 12 imaging modalities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"omnimmi":{"id":"omnimmi","name":"OmniMMI","topic":"Multimodal interaction","description":"OmniMMI benchmark for multimodal interaction across video streams.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"omnispatial":{"id":"omnispatial","name":"OmniSpatial","topic":"Spatial reasoning","description":"Spatial understanding and reasoning benchmark (OmniSpatial).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Reasoning","Vision"]},"one-sided-statement":{"id":"one-sided-statement","name":"One-Sided Statement","topic":"Safety / jailbreak","description":"Robustness to one-sided statement adversarial prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"oneig-bench-en":{"id":"oneig-bench-en","name":"OneIG-Bench EN","topic":"Text-to-image","description":"OneIG-Bench English subset score for text-to-image generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"oneig-bench-zh":{"id":"oneig-bench-zh","name":"OneIG-Bench ZH","topic":"Text-to-image","description":"OneIG-Bench Chinese subset score for text-to-image generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"onemillionbench":{"id":"onemillionbench","name":"OneMillionBench","topic":"Agentic search / expert tasks","description":"OneMillion Bench: expert-level, economically valuable tasks evaluated with tools (search).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"online-mind2web":{"id":"online-mind2web","name":"Online-Mind2web","topic":"Web automation","description":"Online web automation and task execution benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Web","Text","Agents"]},"open-rewrite":{"id":"open-rewrite","name":"Open Rewrite","topic":"Instruction following","description":"Rewrite benchmark assessing open-ended editing and directive-following quality.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Writing","Text"]},"openai-internal-advanced-cyber-completion-rate":{"id":"openai-internal-advanced-cyber-completion-rate","name":"OpenAI Internal Advanced Cybersecurity Completion Rate","topic":"Cybersecurity","description":"OpenAI internal eval (Daybreak announcement): share of advanced, dual-use cybersecurity requests (exploit chains, auth bypass, privilege escalation) the model completes.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Security","Text"]},"openai-internal-broken-search-nondisclosure":{"id":"openai-internal-broken-search-nondisclosure","name":"OpenAI Internal Broken Search Tool Non-Disclosure","topic":"Safety","description":"OpenAI internal alignment eval: rate of failing to disclose a broken search tool (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Alignment","Text"]},"openai-internal-circumvention":{"id":"openai-internal-circumvention","name":"OpenAI Internal Circumvention","topic":"Safety","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: rate of circumventing safeguards (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Alignment","Text"]},"openai-internal-coding-deception":{"id":"openai-internal-coding-deception","name":"OpenAI Internal Coding Deception","topic":"Safety","description":"OpenAI internal alignment eval: fraction of coding-agent answers with detected deception (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Alignment","Code","Text"]},"openai-internal-cua-safety":{"id":"openai-internal-cua-safety","name":"OpenAI Internal Computer-Use Safety","topic":"Safety","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: rate of unsafe actions during computer use (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Agents","Text"]},"openai-internal-cua-safety-autoreview":{"id":"openai-internal-cua-safety-autoreview","name":"OpenAI Internal Computer-Use Safety (AutoReview)","topic":"Safety","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: unsafe computer-use action rate with AutoReview enabled (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Agents","Text"]},"openai-internal-data-science-tasks":{"id":"openai-internal-data-science-tasks","name":"OpenAI Internal Data Science Tasks","topic":"Data science","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: data-science tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Data Science","Text"]},"openai-internal-db-migration-tasks":{"id":"openai-internal-db-migration-tasks","name":"OpenAI Internal Database Migration Tasks","topic":"Software engineering","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: database migration coding tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"openai-internal-design-tasks":{"id":"openai-internal-design-tasks","name":"OpenAI Internal Design Tasks","topic":"Design","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: professional design tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Professional","Text"]},"openai-internal-expert-swe":{"id":"openai-internal-expert-swe","name":"OpenAI Internal Expert-SWE","topic":"Software engineering","description":"OpenAI internal eval (GPT-5.5 announcement): expert-level, long-horizon software-engineering tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"openai-internal-factual-error-rate":{"id":"openai-internal-factual-error-rate","name":"OpenAI Internal Factual Error Rate (Difficult Prompts)","topic":"Factuality","description":"OpenAI internal factuality eval on de-identified ChatGPT conversations where users flagged factual errors; error rate (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Text"]},"openai-internal-hallucination":{"id":"openai-internal-hallucination","name":"OpenAI Internal Hallucination","topic":"Hallucination","description":"OpenAI internal eval reported in the GPT-6 Astra announcement: hallucination rate (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Hallucination","Text"]},"openai-internal-investment-banking-modeling":{"id":"openai-internal-investment-banking-modeling","name":"OpenAI Internal Investment Banking Modeling Tasks","topic":"Finance","description":"OpenAI internal eval (GPT-5.4/GPT-5.5 announcements): investment-banking spreadsheet modeling tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Professional","Text"]},"openai-internal-management-consulting":{"id":"openai-internal-management-consulting","name":"OpenAI Internal Management Consulting Tasks","topic":"Professional work","description":"OpenAI internal eval (GPT-5.6 announcement): management-consulting tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Professional","Text"]},"openai-internal-research-debugging":{"id":"openai-internal-research-debugging","name":"OpenAI Internal Research Debugging Evaluation","topic":"AI research automation","description":"OpenAI internal eval (GPT-5.6 announcement): debugging real AI research systems.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Research","Text"]},"openai-internal-reviewer-bypass":{"id":"openai-internal-reviewer-bypass","name":"OpenAI Internal Reviewer Bypass Attempts","topic":"Safety","description":"OpenAI internal alignment eval: rate of attempts to bypass a reviewer (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Alignment","Agents","Text"]},"openai-internal-unauthorized-agent-interaction":{"id":"openai-internal-unauthorized-agent-interaction","name":"OpenAI Internal Unauthorized Agent Interaction","topic":"Safety","description":"OpenAI internal alignment eval: rate of unauthorized interaction with other agents (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Alignment","Agents","Text"]},"openai-internal-vuln-discovery-report-writing":{"id":"openai-internal-vuln-discovery-report-writing","name":"OpenAI Internal Vulnerability Discovery and Report Writing","topic":"Cybersecurity","description":"OpenAI internal eval (Daybreak announcement): open-ended vulnerability finding in a repo with a known vulnerability, PoC and report quality.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"openai-internal-warning-circumvention":{"id":"openai-internal-warning-circumvention","name":"OpenAI Internal Warning Circumvention","topic":"Safety","description":"OpenAI internal alignment eval: rate of circumventing warnings (lower is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Alignment","Agents","Text"]},"openai-internal-zero-day-discovery":{"id":"openai-internal-zero-day-discovery","name":"OpenAI Internal Zero-Day Discovery Eval","topic":"Cybersecurity","description":"OpenAI internal eval (Daybreak announcement): find zero-day vulnerabilities in current open-source releases, build max-impact PoCs and write-ups; graded on severity, impact and calibration.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"openbookqa":{"id":"openbookqa","name":"OpenBookQA","topic":"Science QA","description":"Open-book multiple choice science questions with supporting facts.","url":"https://github.com/allenai/OpenBookQA","stars":"128","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","STEM","Science","Text"]},"openeqa":{"id":"openeqa","name":"OpenEQA","topic":"Embodied QA","description":"Open-vocabulary embodied question answering over episodic memory of indoor scenes.","url":"https://open-eqa.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":""},"openrewrite-eval":{"id":"openrewrite-eval","name":"OpenRewrite-Eval","topic":"Rewrite quality","description":"OpenRewrite evaluation; micro-averaged RougeL.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"rougel","higher_is_better":true,"classification":["Writing","Text"]},"openscore-string-quartets":{"id":"openscore-string-quartets","name":"OpenScore String Quartets","topic":"Optical music recognition","description":"Optical music recognition on OpenScore string quartet scores; score is 1 - OMR normalized edit distance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Vision","OCR","Multimodal"]},"optmath":{"id":"optmath","name":"OptMATH","topic":"Math optimization reasoning","description":"OptMATH benchmark targeting challenging math optimization and problem-solving tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"order-15-items":{"id":"order-15-items","name":"Order 15 Items","topic":"List ordering","description":"Ordering benchmark requiring models to sequence 15 items correctly.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Memory","Text"]},"order-30-items":{"id":"order-30-items","name":"Order 30 Items","topic":"List ordering (long)","description":"Ordering benchmark requiring models to sequence 30 items correctly.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Memory","Text"]},"osworld":{"id":"osworld","name":"OSWorld","topic":"GUI agents","description":"Agentic GUI task completion and grounding on desktop environments.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld-2.0":{"id":"osworld-2.0","name":"OSWorld 2.0","topic":"GUI agents","description":"OSWorld 2.0 long-horizon computer-use benchmark on a live Ubuntu VM; partial-credit score.","url":"https://arxiv.org/abs/2606.29537","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld-2.0-offline":{"id":"osworld-2.0-offline","name":"OSWorld 2.0 (Offline)","topic":"GUI agents","description":"OSWorld 2.0 offline set (v2026.08.08), partial-credit score on long-horizon computer-use workflows without internet access.","url":"https://osworld-v2.xlang.ai/","stars":"","updated_at":"","leaderboard":"https://osworld-v2.xlang.ai/","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"osworld-2.0-strict":{"id":"osworld-2.0-strict","name":"OSWorld 2.0 (Strict)","topic":"GUI agents","description":"OSWorld 2.0 strict pass rate (full task success).","url":"https://arxiv.org/abs/2606.29537","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld-2.1":{"id":"osworld-2.1","name":"OSWorld 2.1","topic":"GUI agents","description":"OSWorld 2.1 long-horizon computer-use benchmark; partial-credit score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld-2.1-offline":{"id":"osworld-2.1-offline","name":"OSWorld 2.1 (Offline)","topic":"GUI agents","description":"OSWorld 2.1 offline subset (82 tasks), partial-credit score on long-horizon computer-use workflows without internet access.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"osworld-2.1-strict":{"id":"osworld-2.1-strict","name":"OSWorld 2.1 (Strict)","topic":"GUI agents","description":"OSWorld 2.1 strict pass rate (full task success).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld-g":{"id":"osworld-g","name":"OSWorld-G","topic":"GUI agents","description":"OSWorld-G center accuracy (no_refusal).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld-verified":{"id":"osworld-verified","name":"OSWorld Verified","topic":"GUI agents","description":"Verified subset of OSWorld GUI agent benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"osworld2":{"id":"osworld2","name":"OSWorld2","topic":"GUI agents","description":"Second-generation OSWorld GUI agent benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"ovbench":{"id":"ovbench","name":"OVBench","topic":"Open-vocabulary streaming","description":"Open-vocabulary benchmark for streaming video understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Streaming","Vision"]},"ovisocr-kie-inhouse":{"id":"ovisocr-kie-inhouse","name":"OvisOCR In-house KIE Bench","topic":"Key information extraction","description":"OvisOCR team in-house key information extraction benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"ovobench":{"id":"ovobench","name":"OVOBench","topic":"Streaming video QA","description":"Streaming video QA benchmark with open-vocabulary queries.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Streaming","QA","Vision"]},"p3m-500-np-mad":{"id":"p3m-500-np-mad","name":"P3M-500-NP (Box) Matting (MAD)","topic":"Image matting","description":"P3M-500-NP (Box): box-prompted portrait matting; mean absolute difference x10^3 (lower is better).","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"mad","higher_is_better":false,"classification":["Vision","Multimodal","Perception"]},"pagentbench":{"id":"pagentbench","name":"PAgentBench","topic":"Personal assistant agents","description":"iFlytek in-house Personal Assistant Benchmark: task management, file operations and system interactions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"paper-bench-code-dev":{"id":"paper-bench-code-dev","name":"PaperBench Code-Dev","topic":"Code understanding","description":"PaperBench developer subset measuring code reasoning accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"paperbench":{"id":"paperbench","name":"PaperBench","topic":"Research paper understanding","description":"Benchmark for understanding and reasoning over research papers.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Document","Reasoning","Text"]},"parsebench":{"id":"parsebench","name":"ParseBench","topic":"Document parsing","description":"Document-parsing benchmark covering tables, charts, content faithfulness, semantic formatting and visual grounding; overall over 5 categories.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document Understanding","Vision"]},"part-affordance":{"id":"part-affordance","name":"Part-Affordance","topic":"Embodied reasoning","description":"Part-level affordance grounding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"pathvqa":{"id":"pathvqa","name":"PathVQA","topic":"Medical QA","description":"Pathology visual question answering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"paws-x":{"id":"paws-x","name":"PAWS-X","topic":"Paraphrase identification","description":"Cross-lingual adversarial paraphrase identification.","url":"https://github.com/google-research-datasets/paws","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Classification","Text"]},"penguinscrolls":{"id":"penguinscrolls","name":"PenguinScrolls","topic":"Long-context eval","description":"Long-context reading comprehension and reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"perceptionbench":{"id":"perceptionbench","name":"PerceptionBench","topic":"Visual perception","description":"Moonshot benchmark of atomic visual perception capabilities in multimodal LLMs.","url":"https://github.com/MoonshotAI/PerceptionBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"perceptiontest":{"id":"perceptiontest","name":"Perception Test","topic":"Video understanding","description":"DeepMind Perception Test: multimodal video benchmark of perception and reasoning skills (MC video QA).","url":"https://github.com/google-deepmind/perception_test","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":""},"permemsafe":{"id":"permemsafe","name":"PerMemSafe","topic":"Memory safety","description":"Personal-memory safety benchmark: appropriate use and protection of stored user memories.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"persona-modulation":{"id":"persona-modulation","name":"Persona Modulation","topic":"Safety / jailbreak","description":"Robustness to persona modulation jailbreak attempts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"personal-info-leak-fs":{"id":"personal-info-leak-fs","name":"PersonalInfoLeak (few-shot)","topic":"Privacy / data protection","description":"Evaluates resistance to leaking personal or sensitive information under few-shot prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Security","Privacy","Text"]},"personamem":{"id":"personamem","name":"PersonaMem","topic":"Personalization","description":"Evaluates whether models track evolving user personas and preferences across long interaction histories.","url":"https://github.com/bowen-upenn/PersonaMem","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2504.14225","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"perspective-bench":{"id":"perspective-bench","name":"Perspective Bench","topic":"Value alignment","description":"Re-alignment score over 80 prompts measuring adherence to the Public AI Constitution (balanced perspectives).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Alignment","Text"]},"phishnchips":{"id":"phishnchips","name":"PhishNChips","topic":"Phishing detection","description":"Phishing vs legitimate email-and-link decisions; accuracy (Jev Decision Index suite, posed as typed decisions).","url":"https://huggingface.co/spaces/multimodalart/jev-decision-index","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Classification","Text"]},"phybench":{"id":"phybench","name":"PHYBench","topic":"Physics reasoning","description":"Physics reasoning and calculation benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["STEM","Physics","Reasoning","Text"]},"physical-safety":{"id":"physical-safety","name":"PhysicalSafety","topic":"Safety","description":"Physical harm prevention prompts assessing safety policies and refusal.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"phyx":{"id":"phyx","name":"PhyX","topic":"Physics reasoning (multimodal)","description":"Multimodal physics reasoning benchmark (PhyX).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Physics","Reasoning","STEM","Vision"]},"pinchbench":{"id":"pinchbench","name":"PinchBench","topic":"Agentic assistant tasks","description":"Benchmark of OpenClaw agent task success across real-world personal-assistant tasks.","url":"https://pinchbench.com/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"pinchbench-avg":{"id":"pinchbench-avg","name":"PinchBench (Avg)","topic":"Agent tasks (OpenClaw)","description":"PinchBench OpenClaw-style agent task benchmark, average score across runs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"pinchbench-best":{"id":"pinchbench-best","name":"PinchBench (Best)","topic":"Agent tasks (OpenClaw)","description":"PinchBench OpenClaw-style agent task benchmark, best-of-runs score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"piobench":{"id":"piobench","name":"PIOBench","topic":"Embodied reasoning","description":"Pointing / interaction-object grounding benchmark for embodied VLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"piqa":{"id":"piqa","name":"PIQA","topic":"Physical commonsense","description":"Physical commonsense about everyday tasks and object affordances.","url":"https://github.com/ybisk/piqa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"pixmo-points":{"id":"pixmo-points","name":"PixMo-Points","topic":"Embodied reasoning","description":"Pointing accuracy on the PixMo-Points evaluation set.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"pixmocount":{"id":"pixmocount","name":"PixmoCount","topic":"Visual counting","description":"Counting objects/instances in images (PixmoCount).","url":"https://huggingface.co/datasets/allenai/pixmo-count","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2409.17146v2","data":"https://huggingface.co/datasets/allenai/pixmo-count","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Counting","Multimodal","Vision"]},"plawbench":{"id":"plawbench","name":"PLawBench","topic":"Legal reasoning","description":"Legal-practice benchmark; Qwen evaluates it with gemini-3.1-pro-preview as judge.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Text"]},"pmc-vqa":{"id":"pmc-vqa","name":"PMC-VQA","topic":"Medical VQA","description":"PubMed Central visual question answering benchmark for biomedical images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Multimodal","QA","Vision"]},"point-bench":{"id":"point-bench","name":"Point-Bench","topic":"Pointing and counting","description":"Benchmark for pointing and counting objects in images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Counting","Multimodal","Vision"]},"pointarena":{"id":"pointarena","name":"PointArena","topic":"Visual grounding","description":"Language-guided pointing benchmark for multimodal models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"polymath":{"id":"polymath","name":"PolyMATH","topic":"Math reasoning","description":"Polyglot mathematics benchmark assessing cross-topic math reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"pop909-cl-decision":{"id":"pop909-cl-decision","name":"POP909-CL (decision)","topic":"Music understanding","description":"Chord-label decisions over POP909 symbolic music; accuracy. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/spaces/multimodalart/jev-decision-index","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Music","Decision","Text"]},"pope":{"id":"pope","name":"POPE","topic":"Hallucination detection","description":"Vision-language hallucination benchmark focusing on object existence verification.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/AoiDragon/POPE","paper":"https://arxiv.org/abs/2305.10355","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Hallucination","Multimodal","Vision"]},"popqa":{"id":"popqa","name":"PopQA","topic":"Knowledge / QA","description":"Open-domain popular culture question answering benchmark testing long-tail factual recall.","url":"https://github.com/facebookresearch/PopQA","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/facebookresearch/PopQA","paper":"https://arxiv.org/abs/2211.07239","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"posttrainbench":{"id":"posttrainbench","name":"PostTrainBench","topic":"Post-training automation","description":"Measures how well AI agents can post-train base LLMs under fixed compute/time constraints; average score across AIME 2025, BFCL, GPQA Main, GSM8K, and HumanEval.","url":"https://posttrainbench.com/","stars":"","updated_at":"","leaderboard":"https://posttrainbench.com/","github":"https://github.com/aisa-group/PostTrainBench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Training","Reasoning","Text"]},"posttrainbench-lite":{"id":"posttrainbench-lite","name":"PostTrainBench Lite","topic":"Post-training automation","description":"Lite variant of PostTrainBench (agents post-training base LLMs under fixed compute).","url":"https://posttrainbench.com/","stars":"","updated_at":"","leaderboard":"https://posttrainbench.com/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Training","Text"]},"ppm-100-mad":{"id":"ppm-100-mad","name":"PPM-100 (Box) Matting (MAD)","topic":"Image matting","description":"PPM-100 (Box): box-prompted portrait matting; mean absolute difference x10^3 (lower is better).","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"mad","higher_is_better":false,"classification":["Vision","Multimodal","Perception"]},"prbench-finance":{"id":"prbench-finance","name":"PRBench Finance","topic":"Professional reasoning (finance)","description":"PRBench finance subset: rubric-graded professional reasoning tasks in finance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Reasoning","Text"]},"prbench-legal":{"id":"prbench-legal","name":"PRBench Legal","topic":"Professional reasoning (legal)","description":"PRBench legal subset: rubric-graded professional reasoning tasks in law.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Reasoning","Text"]},"prdbench":{"id":"prdbench","name":"PRDBench","topic":"Agentic coding","description":"Product Requirements Document benchmark for evaluating agentic coding capabilities.","url":"","stars":"","updated_at":"2026-01-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"presentbench":{"id":"presentbench","name":"PresentBench","topic":"Agentic office work (slides)","description":"Benchmark for generating presentation decks from source materials, scored with rubrics.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"procbench":{"id":"procbench","name":"ProcBench","topic":"Procedural reasoning","description":"Procedural reasoning benchmark evaluating step-by-step logical reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"programbench":{"id":"programbench","name":"ProgramBench","topic":"Software engineering","description":"ProgramBench (200 instances): agents re-implement complete programs from scratch; score as reported in GLM-5.2 release.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"programbench-almost-solved":{"id":"programbench-almost-solved","name":"ProgramBench (Almost Solved)","topic":"Software engineering","description":"ProgramBench share of instances that are almost solved (near-complete test pass rate).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"prompt-extraction-robustness":{"id":"prompt-extraction-robustness","name":"PromptExtractionRobustness","topic":"Prompt security","description":"Measures robustness against prompt extraction and jailbreak attempts to reveal hidden instructions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Security","Robustness","Text"]},"prompt-injection":{"id":"prompt-injection","name":"PromptInjection","topic":"Safety / jailbreak","description":"Resistance to prompt injection techniques that override instructions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"prontoqa":{"id":"prontoqa","name":"PrOntoQA","topic":"Logical reasoning","description":"Probing ontological reasoning via question answering.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"proofbench-advanced":{"id":"proofbench-advanced","name":"ProofBench Advanced","topic":"Mathematical proofs (advanced)","description":"Advanced mathematical proof benchmark covering complex theorem proving tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"proofbench-basic":{"id":"proofbench-basic","name":"ProofBench Basic","topic":"Mathematical proofs","description":"Entry-level mathematical proof benchmarking set.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"proofnet-test":{"id":"proofnet-test","name":"ProofNet (Test)","topic":"Formal theorem proving","description":"ProofNet test split: undergraduate-level mathematics theorems formalized in Lean 4; score = share proved.","url":"https://github.com/zhangir-azerbayev/ProofNet","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"protocolqa":{"id":"protocolqa","name":"ProtocolQA","topic":"Protocol understanding and QA","description":"Protocol question answering benchmark evaluating understanding of scientific protocols and procedures.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Science","Text"]},"proverbench":{"id":"proverbench","name":"ProverBench","topic":"Formal theorem proving","description":"ProverBench (DeepSeek-Prover-V2): 325 formalized problems incl. AIME and textbook problems in Lean 4; score = share proved.","url":"https://huggingface.co/datasets/deepseek-ai/DeepSeek-ProverBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"pubmedqa":{"id":"pubmedqa","name":"PubMedQA","topic":"Biomedical QA","description":"Yes/no/maybe question answering over PubMed abstracts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Medical","Text"]},"pubtabnet":{"id":"pubtabnet","name":"PubTabNet","topic":"Table recognition","description":"Image-based table recognition benchmark from PubMed Central articles; TEDS score on validation set.","url":"https://github.com/ibm-aur-nlp/PubTabNet","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Vision","Document"]},"puredocbench":{"id":"puredocbench","name":"PureDocBench","topic":"Document parsing","description":"Document parsing benchmark (reported in OvisOCR3 release).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Document","Vision"]},"putnam-2025":{"id":"putnam-2025","name":"Putnam 2025","topic":"Math (competition, proof writing)","description":"Natural-language proof solutions to the 2025 William Lowell Putnam Mathematical Competition (12 problems, 10 points each); score = points out of 120.","url":"https://maa.org/putnam/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"points","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"putnambench":{"id":"putnambench","name":"PutnamBench","topic":"Formal theorem proving","description":"672 Putnam competition problems formalized in Lean 4; score = share of problems proved.","url":"https://github.com/trishullab/PutnamBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Formal Proofs","Text"]},"q-bench":{"id":"q-bench","name":"Q-Bench","topic":"Low-level visual perception","description":"Q-Bench (A1 perception, test split) evaluating MLLMs on low-level visual attributes and image quality questions.","url":"https://q-future.github.io/Q-Bench/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/Q-Future/Q-Bench","paper":"https://arxiv.org/abs/2309.14181","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Perception","Multimodal","Vision"]},"q-spatial-bench":{"id":"q-spatial-bench","name":"Q-Spatial-Bench","topic":"Embodied reasoning","description":"Quantitative spatial reasoning (metric distances/sizes) benchmark for VLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"qivd":{"id":"qivd","name":"QIVD","topic":"Audio-visual QA","description":"Qualcomm Interactive Video Dataset: situated audio-visual question answering on live-style video with speech questions; accuracy.","url":"https://thinkingmachines.ai/blog/interaction-models/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Vision","Video"]},"qrecc":{"id":"qrecc","name":"QReCC","topic":"Conversational QA","description":"Open-domain conversational QA with question rewriting (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"quac":{"id":"quac","name":"QuAC","topic":"Conversational QA","description":"Question answering in context.","url":"https://quac.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"quality":{"id":"quality","name":"QuALITY","topic":"Long-context reading comprehension","description":"Long-document multiple-choice reading comprehension benchmark.","url":"https://github.com/nyu-mll/quality","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"https://github.com/nyu-mll/quality","paper":"https://arxiv.org/abs/2112.08608","modality":"text","metric":"log_acc","higher_is_better":true,"classification":["Long Context","Reading Comprehension","Text"]},"query-sufficiency":{"id":"query-sufficiency","name":"Query Sufficiency","topic":"Legal query triage","description":"Judging whether a legal research query contains sufficient information to answer (Thomson Reuters eval).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Legal","Text"]},"qvhighlights":{"id":"qvhighlights","name":"QVHighlights","topic":"Video temporal grounding","description":"Video moment retrieval / highlight detection benchmark; R1@0.5.","url":"https://github.com/jayleicn/moment_detr","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"qwenclawbench":{"id":"qwenclawbench","name":"QwenClawBench","topic":"Agent tasks (OpenClaw)","description":"Real-user-distribution Claw (OpenClaw-style) agent benchmark released by Qwen.","url":"https://github.com/SKYLENAGE-AI/QwenClawBench","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/SKYLENAGE-AI/QwenClawBench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"r-bench":{"id":"r-bench","name":"R-Bench","topic":"Robustness (multimodal)","description":"R-Bench: robustness of LMMs to real-world image corruptions (discriminative subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2410.05474","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Robustness","Vision"]},"race":{"id":"race","name":"RACE","topic":"Reading comprehension","description":"English exams for middle and high school.","url":"http://www.cs.cmu.edu/~glai1/data/race/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Text"]},"race-high":{"id":"race-high","name":"RACE-High","topic":"Reading comprehension","description":"RACE reading-comprehension questions from Chinese high-school English exams.","url":"https://www.cs.cmu.edu/~glai1/data/race/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1704.04683","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Text"]},"race-middle":{"id":"race-middle","name":"RACE-Middle","topic":"Reading comprehension","description":"RACE reading-comprehension questions from Chinese middle-school English exams.","url":"https://www.cs.cmu.edu/~glai1/data/race/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1704.04683","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Text"]},"ragtruth":{"id":"ragtruth","name":"RAGTruth","topic":"Hallucination detection","description":"Response-level hallucination detection for RAG answers; F1 on the hallucinated class.","url":"https://github.com/ParticleMedia/RAGTruth","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Hallucination","Text"]},"random-complex-tasks":{"id":"random-complex-tasks","name":"Random Complex Tasks","topic":"Agentic tasks (random)","description":"Randomly constructed complex task environments for agent generalization.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"real5-omnidocbench":{"id":"real5-omnidocbench","name":"Real5-OmniDocBench","topic":"Document parsing (real-world distortions)","description":"OmniDocBench v1.5 pages re-captured under five real-world conditions (scanning, warping, screen photography, illumination, skew); Overall score.","url":"https://huggingface.co/datasets/PaddlePaddle/Real5-OmniDocBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2601.21957","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Multimodal","OCR","Vision"]},"realbench":{"id":"realbench","name":"Realbench","topic":"Web browsing","description":"Real-world browsing and QA benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Web","Text"]},"realkie-fcc-verified":{"id":"realkie-fcc-verified","name":"RealKIE-FCC Verified","topic":"Key information extraction","description":"Human-verified version of the RealKIE FCC document key-information-extraction benchmark (ALNS).","url":"https://huggingface.co/datasets/amazon-agi/RealKIE-FCC-Verified","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Document","Vision","Multimodal"]},"realworldqa":{"id":"realworldqa","name":"RealWorldQA","topic":"Real-world visual QA","description":"Visual question answering with real-world images and scenarios.","url":"https://realworldqa.github.io/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/ZhuoyuWei/RealWorldQAPlatform","paper":"https://arxiv.org/abs/2410.14332","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multimodal","QA","Vision"]},"reasonseg-val":{"id":"reasonseg-val","name":"ReasonSeg (val)","topic":"Reasoning segmentation","description":"ReasonSeg val reasoning segmentation, gIoU.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring","Reasoning"]},"recruit-bench":{"id":"recruit-bench","name":"Recruit-Bench","topic":"Recruitment / HR tasks","description":"Benchmark of recruiting-domain tasks (resume screening, job matching, candidate communication) used by Nanbeige.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Alignment","Text"]},"ref-l4-rod-f1":{"id":"ref-l4-rod-f1","name":"Ref-L4 Referring Detection (F1)","topic":"Referring expressions","description":"Ref-L4 referring object detection, F1 mean over IoU 0.5:0.95.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"ref-l4-test":{"id":"ref-l4-test","name":"Ref-L4 (test)","topic":"Referring expressions","description":"Ref-L4 referring expression comprehension on the test split.","url":"https://huggingface.co/datasets/JierunChen/Ref-L4","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","data":"https://huggingface.co/datasets/JierunChen/Ref-L4","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Referring","Multimodal","Vision"]},"ref-zom-ciou":{"id":"ref-zom-ciou","name":"Ref-ZOM (cIoU)","topic":"Referring segmentation","description":"Ref-ZOM zero/one/many-target referring segmentation, cIoU.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"ref-zom-rejection":{"id":"ref-zom-rejection","name":"Ref-ZOM (Rejection)","topic":"Referring segmentation","description":"Ref-ZOM rejection rate on non-existent referents.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring","Hallucination"]},"refcoco":{"id":"refcoco","name":"RefCOCO","topic":"Referring expressions","description":"RefCOCO average accuracy at IoU 0.5 (val).","url":"https://huggingface.co/datasets/JierunChen/Ref-L4","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","data":"https://huggingface.co/datasets/JierunChen/Ref-L4","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Referring","Multimodal","Vision"]},"refcoco-m":{"id":"refcoco-m","name":"RefCOCO-M","topic":"Referring expressions","description":"RefCOCO-M: re-annotated RefCOCO with cleaned masks/boxes for referring-expression grounding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Referring","Multimodal","Vision"]},"refcoco-testa-ciou":{"id":"refcoco-testa-ciou","name":"RefCOCO testA Segmentation (cIoU)","topic":"Referring segmentation","description":"RefCOCO testA referring segmentation, cIoU.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"refcocog":{"id":"refcocog","name":"RefCOCOg","topic":"Referring expressions","description":"RefCOCOg average accuracy at IoU 0.5 (val).","url":"https://huggingface.co/datasets/JierunChen/Ref-L4","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","data":"https://huggingface.co/datasets/JierunChen/Ref-L4","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Referring","Multimodal","Vision"]},"refcocog-rod-f1":{"id":"refcocog-rod-f1","name":"RefCOCOg Referring Detection (F1)","topic":"Referring expressions","description":"RefCOCOg test referring object detection (multi-box), F1 mean over IoU 0.5:0.95.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"refcocog-rod-rejection":{"id":"refcocog-rod-rejection","name":"RefCOCOg Referring Detection (Rejection)","topic":"Referring expressions","description":"Rejection rate on non-existent referents for RefCOCOg referring detection.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring","Hallucination"]},"refcocog-test-ciou":{"id":"refcocog-test-ciou","name":"RefCOCOg test Segmentation (cIoU)","topic":"Referring segmentation","description":"RefCOCOg test referring segmentation, cIoU.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"refcocoplus":{"id":"refcocoplus","name":"RefCOCO+","topic":"Referring expressions","description":"RefCOCO+ accuracy at IoU 0.5 on the val split.","url":"https://huggingface.co/datasets/JierunChen/Ref-L4","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","data":"https://huggingface.co/datasets/JierunChen/Ref-L4","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Referring","Multimodal","Vision"]},"refcocoplus-testa-ciou":{"id":"refcocoplus-testa-ciou","name":"RefCOCO+ testA Segmentation (cIoU)","topic":"Referring segmentation","description":"RefCOCO+ testA referring segmentation, cIoU.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"refspatialbench":{"id":"refspatialbench","name":"RefSpatialBench","topic":"Spatial reasoning","description":"Reference spatial understanding benchmark covering spatial grounding tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Multimodal","Vision"]},"refusal-suppression":{"id":"refusal-suppression","name":"Refusal Suppression","topic":"Safety / jailbreak","description":"Resistance to adversarial prompts suppressing refusal behaviors.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"refusalbench":{"id":"refusalbench","name":"RefusalBench","topic":"Safety / refusal","description":"Safety-oriented refusal and policy adherence benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"remi":{"id":"remi","name":"ReMI","topic":"Multimodal reasoning","description":"Reasoning over multimodal inputs (ReMI).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"repobench":{"id":"repobench","name":"RepoBench","topic":"Code understanding","description":"Repository-level code comprehension and reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Reasoning","Text"]},"researchrubrics":{"id":"researchrubrics","name":"ResearchRubrics","topic":"Research evaluation","description":"Benchmark evaluating model ability to conduct research and synthesize findings.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Research","Text"]},"rewardbench":{"id":"rewardbench","name":"RewardBench","topic":"Reward modeling","description":"Preference/reward model evaluation benchmark.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"https://huggingface.co/spaces/allenai/reward-bench","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"riemannbench":{"id":"riemannbench","name":"RiemannBench","topic":"Research math","description":"Research-level mathematics benchmark (reported in Anthropic Fable 5/Mythos 5 system card).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"rlhf":{"id":"rlhf","name":"RLHF Benchmark","topic":"Safety alignment","description":"Helpful-Harmless RLHF preference evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Alignment","Text"]},"roboafford":{"id":"roboafford","name":"RoboAfford","topic":"Embodied reasoning","description":"Robotic affordance (object & spatial) grounding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"roborefit":{"id":"roborefit","name":"RoboRefIt","topic":"Embodied reasoning","description":"Referring-expression grounding in cluttered robotic scenes.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"robospatialhome":{"id":"robospatialhome","name":"RoboSpatialHome","topic":"Embodied spatial understanding","description":"RoboSpatialHome benchmark for embodied spatial reasoning in domestic environments.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Embodied","Vision"]},"robotgrounding-inhouse":{"id":"robotgrounding-inhouse","name":"RobotGrounding (in-house)","topic":"Embodied grounding","description":"ACE Robotics in-house robot grounding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision","Robotics"]},"robovqa":{"id":"robovqa","name":"RoboVQA","topic":"Embodied reasoning","description":"Robotics video question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"roo-code-evals":{"id":"roo-code-evals","name":"Roo Code Evals","topic":"Code assistant eval","description":"Community-maintained coding evals and leaderboard by Roo Code.","url":"https://roocode.com/evals","stars":"","updated_at":"2025-09-09","leaderboard":"https://roocode.com/evals","github":"https://github.com/RooCodeInc/Roo-Code-Evals","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"routerbench-decision":{"id":"routerbench-decision","name":"RouterBench (decision)","topic":"Model routing","description":"Choose which LLM to route a query to; selected quality (quality objective). Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/withmartian/routerbench","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"score","higher_is_better":true,"classification":["Routing","Decision","Text"]},"ruler-100-1m":{"id":"ruler-100-1m","name":"RULER-100 @1M","topic":"Long-context eval","description":"RULER-100 evaluation at a 1M context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-100-256k":{"id":"ruler-100-256k","name":"RULER-100 @256k","topic":"Long-context eval","description":"RULER-100 evaluation at a 256k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-100-512k":{"id":"ruler-100-512k","name":"RULER-100 @512k","topic":"Long-context eval","description":"RULER-100 evaluation at a 512k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-128k":{"id":"ruler-128k","name":"Ruler 128k","topic":"Long-context eval","description":"RULER benchmark at 128k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-16k":{"id":"ruler-16k","name":"Ruler 16k","topic":"Long-context eval","description":"RULER benchmark at 16k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-1m":{"id":"ruler-1m","name":"Ruler 1M","topic":"Long-context eval","description":"RULER benchmark at 1M context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-2048k":{"id":"ruler-2048k","name":"Ruler 2048k","topic":"Long-context eval","description":"RULER benchmark at 2048k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-256k":{"id":"ruler-256k","name":"Ruler 256k","topic":"Long-context eval","description":"RULER benchmark at 256k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-32k":{"id":"ruler-32k","name":"Ruler 32k","topic":"Long-context eval","description":"RULER benchmark at 32k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-4k":{"id":"ruler-4k","name":"Ruler 4k","topic":"Long-context eval","description":"RULER benchmark at 4k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-512k":{"id":"ruler-512k","name":"Ruler 512k","topic":"Long-context eval","description":"RULER benchmark at 512k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-64k":{"id":"ruler-64k","name":"Ruler 64k","topic":"Long-context eval","description":"RULER benchmark at 64k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"ruler-8k":{"id":"ruler-8k","name":"Ruler 8k","topic":"Long-context eval","description":"RULER benchmark at 8k context window.","url":"https://github.com/allenai/ruler","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.06654","modality":"text","metric":"percent","higher_is_better":true,"classification":["Long Context","Text"]},"rw-search":{"id":"rw-search","name":"RW Search","topic":"Agentic search","description":"Real-world search benchmark evaluating retrieval and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Search","Text"]},"saas-bench":{"id":"saas-bench","name":"SaaS-Bench","topic":"Agentic SaaS tasks","description":"Benchmark of agents completing tasks in SaaS application environments (as reported in the Kimi K3 release).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"safetybench":{"id":"safetybench","name":"SafetyBench","topic":"Safety","description":"Multiple-choice safety evaluation across 7 categories in Chinese and English.","url":"https://github.com/thu-coai/SafetyBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2309.07045","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"salad-bench":{"id":"salad-bench","name":"SALAD-Bench","topic":"Safety alignment","description":"Safety Alignment and Dangerous-behavior benchmark evaluating harmful assistance and refusal consistency.","url":"https://github.com/allenai/salad-bench","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/allenai/salad-bench","paper":"https://arxiv.org/abs/2410.08628","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Text"]},"sarena-icon":{"id":"sarena-icon","name":"SArena (Icon)","topic":"SVG generation","description":"SVG Arena benchmark for icon generation evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Generation","Vision"]},"sat":{"id":"sat","name":"SAT","topic":"Spatial reasoning","description":"Spatial Aptitude Training benchmark of static and dynamic spatial reasoning questions for VLMs.","url":"https://arxiv.org/abs/2412.07755","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2412.07755","modality":"visual","metric":"percent","higher_is_better":true,"classification":""},"sat-math":{"id":"sat-math","name":"SAT Math","topic":"Math (exam)","description":"SAT mathematics questions (DeepSeekMath / Minerva-style evaluation).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2402.03300","modality":"text","metric":"percent","higher_is_better":true,"classification":["Exams","Math","Text"]},"sata-bench":{"id":"sata-bench","name":"SATA-Bench","topic":"Multiple-answer QA","description":"Select-all-that-apply multiple-choice questions; case exact accuracy (Jev Decision Index suite, posed as typed decisions).","url":"https://huggingface.co/datasets/sata-bench/sata-bench","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"scale-ai-multi-challenge":{"id":"scale-ai-multi-challenge","name":"Scale AI Multi Challenge","topic":"Chat & instruction following","description":"Scale AI Multi Challenge crowd-evaluated instruction following benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"scicode":{"id":"scicode","name":"SciCode (sub)","topic":"Code","description":"SciCode subset score (sub).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"scicode-main":{"id":"scicode-main","name":"SciCode (main)","topic":"Code","description":"SciCode main score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"scienceqa":{"id":"scienceqa","name":"ScienceQA","topic":"Science QA (multimodal)","description":"Multiple-choice science questions with images, diagrams, and text context.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Science","Multimodal","Vision","Text"]},"sciq":{"id":"sciq","name":"SciQ","topic":"Science QA","description":"Multiple choice science questions.","url":"https://allenai.org/data/sciq","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","STEM","Science","Text"]},"scireasoner":{"id":"scireasoner","name":"SciReasoner","topic":"Scientific reasoning","description":"Scientific reasoning benchmark evaluating multimodal AI models on scientific tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Science","Reasoning","STEM","Vision"]},"scires-frontiermath-tier-1-3":{"id":"scires-frontiermath-tier-1-3","name":"SciRes FrontierMath Tier 1-3","topic":"Math (frontier)","description":"SciRes FrontierMath benchmark covering tiers 1-3.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"scires-frontiermath-tier-4":{"id":"scires-frontiermath-tier-4","name":"SciRes FrontierMath Tier 4","topic":"Math (frontier)","description":"SciRes FrontierMath benchmark covering tier 4.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","STEM","Text"]},"screenqa-complex":{"id":"screenqa-complex","name":"ScreenQA Complex","topic":"GUI QA","description":"Complex ScreenQA benchmark accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","QA","Multimodal","Vision"]},"screenqa-short":{"id":"screenqa-short","name":"ScreenQA Short","topic":"GUI QA","description":"Short-form ScreenQA benchmark accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","QA","Multimodal","Vision"]},"screenspot":{"id":"screenspot","name":"ScreenSpot","topic":"Screen UI locators","description":"Center accuracy on ScreenSpot.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Multimodal","Vision"]},"screenspot-pro":{"id":"screenspot-pro","name":"ScreenSpot-Pro","topic":"Screen UI locators","description":"Average center accuracy on ScreenSpot-Pro.","url":"","stars":"","updated_at":"","leaderboard":"https://gui-agent.github.io/grounding-leaderboard/","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Multimodal","Vision"]},"screenspot-pro-with-tools":{"id":"screenspot-pro-with-tools","name":"ScreenSpot-Pro (with tools)","topic":"Screen UI locators (with tools)","description":"ScreenSpot-Pro evaluated with tool access (e.g. zoom/crop).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Multimodal","Tools","Vision"]},"screenspot-v2":{"id":"screenspot-v2","name":"ScreenSpot-v2","topic":"Screen UI locators","description":"Center accuracy on ScreenSpot-v2.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Multimodal","Vision"]},"seal-0":{"id":"seal-0","name":"SEAL-0","topic":"Agentic web search","description":"Evaluation of multi-step browsing agents on search, evidence gathering, and synthesis tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Search","Text"]},"sec-bench-pro":{"id":"sec-bench-pro","name":"SEC-Bench Pro","topic":"Cybersecurity","description":"Proof-of-concept exploit generation for vulnerabilities in complex software.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"seccodebench":{"id":"seccodebench","name":"SecCodeBench","topic":"Secure code generation","description":"Benchmark evaluating secure code generation capabilities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Security","Text"]},"seed-bench-2-plus":{"id":"seed-bench-2-plus","name":"SEED-Bench-2-Plus","topic":"Multimodal evaluation","description":"SEED-Bench-2-Plus overall accuracy.","url":"https://huggingface.co/datasets/AILab-CVC/SEED-Bench-2-plus","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.16790","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"seed-bench-img":{"id":"seed-bench-img","name":"SEED-Bench-Img","topic":"Multimodal image understanding","description":"SEED-Bench image-only subset (SEED-Bench-Img).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"seedbench":{"id":"seedbench","name":"SEED-Bench","topic":"Multimodal evaluation","description":"SEED-Bench comprehensive multimodal understanding benchmark evaluating generative comprehension across multiple dimensions.","url":"https://huggingface.co/datasets/AILab-CVC/SEED-Bench","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/AILab-CVC/SEED-Bench","paper":"https://arxiv.org/abs/2307.16125","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision","QA"]},"sfe":{"id":"sfe","name":"SFE","topic":"Multimodal reasoning","description":"Structured factual evaluation for multimodal models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"sgd-x-decision":{"id":"sgd-x-decision","name":"SGD/SGD-X (decision)","topic":"Dialogue state tracking","description":"Schema-Guided Dialogue (SGD-X) intent/slot decisions; macro-F1. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://github.com/google-research-datasets/dstc8-schema-guided-dialogue","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Dialogue","Decision","Text"]},"sgi-bench":{"id":"sgi-bench","name":"SGI-Bench","topic":"Scientific agents","description":"Scientific general intelligence benchmark of agentic scientific research workflows.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Science"]},"sharerobot-affordance":{"id":"sharerobot-affordance","name":"ShareRobot-Affordance","topic":"Embodied affordance","description":"ShareRobot affordance prediction benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision","Robotics"]},"sharerobot-traj":{"id":"sharerobot-traj","name":"ShareRobot-Trajectory","topic":"Embodied reasoning","description":"ShareRobot trajectory prediction benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"showdown":{"id":"showdown","name":"Showdown","topic":"GUI agents","description":"Success rate on the Showdown UI interaction benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"sifo":{"id":"sifo","name":"SIFO","topic":"Instruction following","description":"Single-turn instruction following benchmark.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"sifo-multiturn":{"id":"sifo-multiturn","name":"SIFO Multiturn","topic":"Instruction following","description":"Multi-turn SIFO benchmark for sustained instruction adherence.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"simple-safety-tests":{"id":"simple-safety-tests","name":"SimpleSafetyTests","topic":"Safety / refusal","description":"Simple safety test suite for refusal and policy adherence.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Refusal","Text"]},"simpleqa":{"id":"simpleqa","name":"SimpleQA","topic":"QA","description":"Simple question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"simpleqa-search":{"id":"simpleqa-search","name":"SimpleQA (with search)","topic":"QA","description":"SimpleQA answered with web search / tool use enabled (agentic setting), not closed-book.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"simpleqa-verified":{"id":"simpleqa-verified","name":"SimpleQA Verified","topic":"QA","description":"Verified SimpleQA variant for parametric knowledge accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"simplevqa":{"id":"simplevqa","name":"SimpleVQA","topic":"General VQA","description":"Lightweight visual question answering set with everyday scenes.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"simplevqa-ds":{"id":"simplevqa-ds","name":"SimpleVQA-DS","topic":"General VQA","description":"SimpleVQA variant curated by DeepSeek with everyday image question answering tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"simplevqa-with-tools":{"id":"simplevqa-with-tools","name":"SimpleVQA (With Tools)","topic":"Visual factuality","description":"SimpleVQA visual factuality benchmark evaluated with tool access.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":"Multimodal"},"siqa":{"id":"siqa","name":"Social Interaction QA (SIQA)","topic":"Social commonsense QA","description":"Social Interaction QA benchmark evaluating social commonsense and situational reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Commonsense","Social","QA","Text"]},"skatebench":{"id":"skatebench","name":"SkateBench","topic":"Agentic tasks","description":"Agentic task execution benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"skillsbench":{"id":"skillsbench","name":"SkillsBench","topic":"Agent skills","description":"SkillsBench agentic tasks; Qwen reports avg of 5 runs on the 78-task self-contained subset (API-dependent tasks excluded).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"skillsbench-v1.1":{"id":"skillsbench-v1.1","name":"SkillsBench v1.1","topic":"Agent skills","description":"Public SkillsBench v1.1 benchmark (87 tasks), average score over three runs per task.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"slake":{"id":"slake","name":"SLAKE","topic":"Medical VQA","description":"Semantically-Labeled Knowledge-Enhanced medical visual question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Multimodal","QA","Vision"]},"smolinstruct":{"id":"smolinstruct","name":"SmolInstruct","topic":"Small molecule understanding","description":"Small molecule instruction-following and understanding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chemistry","Science","STEM","Vision"]},"sob":{"id":"sob","name":"SOB (Structured Output Benchmark)","topic":"Structured output","description":"Structured Output Benchmark: extract values into a JSON schema from text, images and audio (5,324 records); value accuracy over exact field matches.","url":"https://interfaze.ai","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"multimodal","metric":"percent","higher_is_better":true,"classification":["Structured Output","Multimodal"]},"socialiqa":{"id":"socialiqa","name":"SocialIQA","topic":"Social commonsense","description":"Social interaction commonsense QA.","url":"https://leaderboard.allenai.org/socialiqa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"sol-execbench":{"id":"sol-execbench","name":"SOL-ExecBench","topic":"GPU kernel optimization","description":"Agents optimize GPU kernels toward hardware speed-of-light limits; mean SOL score (higher is better).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"score","higher_is_better":true,"classification":["Agents","Code","Research"]},"sparbench":{"id":"sparbench","name":"SPAR-Bench","topic":"Spatial reasoning","description":"Spatial perception and reasoning benchmark derived from the SPAR dataset (single- and multi-view).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"spatialsense-vqa":{"id":"spatialsense-vqa","name":"SpatialSense VQA","topic":"Spatial reasoning","description":"SpatialSense spatial-relation recognition posed as VQA (as used in Gemma/PaliGemma evaluations).","url":"https://arxiv.org/abs/1908.02660","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Spatial Reasoning"]},"spatialviz":{"id":"spatialviz","name":"SpatialViz","topic":"Mental visualization","description":"Mental visualization benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Visualization","Vision"]},"speechrole":{"id":"speechrole","name":"SpeechRole","topic":"Spoken role-play","description":"SpeechRole: speech role-playing agent benchmark (aggregate judge score, unbounded scale).","url":"https://arxiv.org/abs/2508.02013","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"score","higher_is_better":true,"classification":["Audio","Voice","Role-play"]},"spider":{"id":"spider","name":"Spider","topic":"Text-to-SQL","description":"Complex text-to-SQL benchmark over cross-domain databases.","url":"https://yale-lily.github.io/spider","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"spider-2.0-lite":{"id":"spider-2.0-lite","name":"Spider 2.0-Lite","topic":"Text-to-SQL","description":"Spider 2.0-Lite enterprise text-to-SQL over real schemas, scored by executing the query (Interfaze reports the 135-task SQLite subset).","url":"https://spider2-sql.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Coding","Text-to-SQL","Text"]},"spiral-bench":{"id":"spiral-bench","name":"Spiral-Bench","topic":"Safety / sycophancy","description":"A LLM-judged benchmark measuring sycophancy and delusion reinforcement.","url":"https://eqbench.com/spiral-bench.html","stars":"","updated_at":"","leaderboard":"https://eqbench.com/spiral-bench.html","github":"https://github.com/EQ-bench","paper":"https://arxiv.org/abs/2312.06281","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Hallucination","Text"]},"spreadsheetbench-2":{"id":"spreadsheetbench-2","name":"SpreadsheetBench 2","topic":"Spreadsheet manipulation","description":"Second version of SpreadsheetBench real-world spreadsheet manipulation tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Office","Text"]},"spreadsheetbench-v1":{"id":"spreadsheetbench-v1","name":"SpreadSheetBench v1","topic":"Spreadsheet manipulation","description":"Real-world spreadsheet manipulation tasks (v1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Office","Text"]},"spreadsheetbench-verified":{"id":"spreadsheetbench-verified","name":"SpreadsheetBench-Verified","topic":"Spreadsheet manipulation","description":"Verified subset of SpreadsheetBench real-world spreadsheet manipulation tasks.","url":"https://spreadsheetbench.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Coding","Text"]},"sqa":{"id":"sqa","name":"SQA","topic":"Table QA","description":"Sequential QA over tables (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"squad-v1.1":{"id":"squad-v1.1","name":"SQuAD v1.1","topic":"Reading comprehension","description":"Extractive QA from Wikipedia articles.","url":"https://github.com/rajpurkar/SQuAD-explorer","stars":"566","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"squad-v2.0":{"id":"squad-v2.0","name":"SQuAD v2.0","topic":"Reading comprehension","description":"Like v1.1 with unanswerable questions.","url":"https://github.com/rajpurkar/SQuAD-explorer","stars":"566","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"sre-bench":{"id":"sre-bench","name":"SRE-Bench","topic":"Cybersecurity","description":"Software reverse engineering of binaries without source code (single-attempt solve rate).","url":"https://arxiv.org/abs/2608.11469v1","stars":"","updated_at":"","leaderboard":"https://arxiv.org/abs/2608.11469v1","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Security","Text"]},"sroie-f1":{"id":"sroie-f1","name":"SROIE (F1)","topic":"OCR","description":"SROIE text grounding and recognition, F1 mean.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","OCR"]},"streamingbench":{"id":"streamingbench","name":"StreamingBench","topic":"Streaming video","description":"Streaming video understanding benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Streaming","Vision"]},"strongreject":{"id":"strongreject","name":"StrongREJECT","topic":"Safety (jailbreak refusal)","description":"StrongREJECT jailbreak benchmark; score = share of forbidden prompts safely handled.","url":"https://arxiv.org/abs/2402.10260","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"stvqa":{"id":"stvqa","name":"ST-VQA","topic":"Scene-text VQA","description":"Scene Text Visual Question Answering (ANLS).","url":"https://rrc.cvc.uab.es/?ch=11","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1905.13648","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","OCR","Vision"]},"sunrgbd":{"id":"sunrgbd","name":"SUNRGBD","topic":"3D scene understanding","description":"SUN RGB-D benchmark for indoor scene understanding from RGB-D imagery.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["3D","Multimodal","Vision"]},"superchem":{"id":"superchem","name":"SuperChem","topic":"Chemistry reasoning","description":"Chemistry reasoning benchmark evaluating text-based chemical knowledge and problem solving.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["STEM","Science","Reasoning","Text"]},"superglue":{"id":"superglue","name":"SuperGLUE","topic":"Multi-task NLU","description":"Suite of harder NLU tasks beyond GLUE.","url":"https://super.gluebenchmark.com/","stars":"","updated_at":"2025-09-08","leaderboard":"https://super.gluebenchmark.com/leaderboard","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"supergpqa":{"id":"supergpqa","name":"SuperGPQA","topic":"Graduate-level QA","description":"Harder GPQA variant assessing advanced graduate-level reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"surds":{"id":"surds","name":"SURDS","topic":"Driving scene spatial reasoning","description":"Spatial understanding and reasoning benchmark on autonomous-driving scenes.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Spatial","Multimodal"]},"svg-bench":{"id":"svg-bench","name":"SVG-Bench","topic":"SVG generation","description":"MiniMax internal SVG generation benchmark (build from scratch or edit from text/image inputs), VLM-verified rendering accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"swe-atlas-qna":{"id":"swe-atlas-qna","name":"SWE-Atlas Codebase QnA","topic":"Codebase understanding","description":"SWE-Atlas Codebase QnA: agentic question answering over real codebases.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"swe-atlas-refactoring":{"id":"swe-atlas-refactoring","name":"SWE-Atlas Refactoring","topic":"Software engineering","description":"SWE-Atlas refactoring track: agentic refactoring tasks over real codebases.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"swe-atlas-test-writing":{"id":"swe-atlas-test-writing","name":"SWE-Atlas Test Writing","topic":"Software testing","description":"SWE-Atlas Test Writing: agentic test-writing tasks over real codebases (Scale AI SWE Atlas).","url":"https://labs.scale.com/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"swe-bench":{"id":"swe-bench","name":"SWE-Bench","topic":"Code repair","description":"Supervised software engineering benchmark across many repos and issues.","url":"https://github.com/princeton-nlp/SWE-bench","stars":"3442","updated_at":"2025-09-05","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-bench-multilingual":{"id":"swe-bench-multilingual","name":"SWE-Bench Multilingual","topic":"Code repair (multilingual)","description":"Multilingual variant of SWE-Bench for issue fixing.","url":"","stars":"","updated_at":"2025-09-05","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-bench-multimodal":{"id":"swe-bench-multimodal","name":"SWE-bench Multimodal","topic":"Software engineering","description":"SWE-bench variant with visual context (screenshots, design mockups) in issue descriptions.","url":"https://arxiv.org/abs/2410.03859","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Multimodal","Vision"]},"swe-bench-openhands":{"id":"swe-bench-openhands","name":"SWE-Bench (OpenHands)","topic":"Code repair","description":"SWE-Bench results using the OpenHands autonomous coding agent.","url":"https://github.com/princeton-nlp/SWE-bench","stars":"3442","updated_at":"2025-09-05","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"swe-bench-pro":{"id":"swe-bench-pro","name":"SWE-Bench Pro","topic":"Software engineering","description":"Full SWE-Bench Pro benchmark for software-engineering agents.","url":"https://www.swebench.com/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"swe-bench-pro-public":{"id":"swe-bench-pro-public","name":"SWE-Bench Pro (Public)","topic":"Software engineering","description":"Public subset of the SWE-Bench Pro benchmark for software-engineering agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"swe-bench-pro-v2":{"id":"swe-bench-pro-v2","name":"SWE-Bench Pro v2","topic":"Software engineering","description":"SWE-Bench Pro v2 (Scale AI) agentic software-engineering tasks; full set.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Coding","Agents","Text"]},"swe-bench-pro-v2-hard":{"id":"swe-bench-pro-v2-hard","name":"SWE-Bench Pro v2 (Hard)","topic":"Software engineering","description":"Hard split of SWE-Bench Pro v2 (Scale AI) for software-engineering agents.","url":"https://scale.com/leaderboard","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"swe-bench-verified":{"id":"swe-bench-verified","name":"SWE-Bench Verified","topic":"Code repair","description":"Verified subset of SWE-Bench for issue fixing.","url":"https://www.swebench.com/","stars":"","updated_at":"2025-09-05","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-dev":{"id":"swe-dev","name":"SWE-Dev","topic":"Code repair","description":"Software engineering development and bug fixing benchmark.","url":"","stars":"","updated_at":"2025-09-05","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-fficiency":{"id":"swe-fficiency","name":"SWE-fficiency","topic":"Code performance optimization","description":"SWE-fficiency: repository-level performance-optimization tasks; agents must speed up real codebases while keeping tests passing.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"swe-interact":{"id":"swe-interact","name":"SWE-Interact","topic":"Interactive software engineering","description":"Interactive SWE benchmark (75 tasks) where the agent must clarify underspecified issues with a simulated user before fixing them.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"swe-lancer":{"id":"swe-lancer","name":"SWE-Lancer","topic":"Code repair (freelance tasks)","description":"Software engineering benchmark using real freelance-style issues.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2502.12115","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-lancer-diamond":{"id":"swe-lancer-diamond","name":"SWE-Lancer Diamond","topic":"Code repair (freelance)","description":"Diamond subset of SWE-Lancer focusing on the hardest freelance-style issues.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-lancer-ic-diamond":{"id":"swe-lancer-ic-diamond","name":"SWE-Lancer IC Diamond","topic":"Code repair (freelance)","description":"Individual Contributor Diamond subset of SWE-Lancer.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-marathon":{"id":"swe-marathon","name":"SWE-Marathon","topic":"Software engineering","description":"Abundant AI's SWE-Marathon: very long-horizon software engineering tasks (1M context, max effort).","url":"https://www.abundant.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"swe-marathon-v1.1":{"id":"swe-marathon-v1.1","name":"SWE-Marathon v1.1","topic":"Software engineering","description":"SWE-Marathon v1.1: very long-horizon software engineering tasks (Abundant AI).","url":"https://www.abundant.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"swe-perf":{"id":"swe-perf","name":"SWE-Perf","topic":"Code repair","description":"Software engineering benchmark focused on performance-oriented fixes.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swe-review":{"id":"swe-review","name":"SWE-Review","topic":"Code review","description":"Software engineering review benchmark for assessing code review quality.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"swebbench":{"id":"swebbench","name":"SWebBench","topic":"Web research agents","description":"iFlytek in-house benchmark for web browsing, information retrieval and research (reported with Spark-X2.6).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Search","Text"]},"switzerland-qa":{"id":"switzerland-qa","name":"Switzerland QA","topic":"Country-specific QA","description":"Question answering focused on Switzerland-specific facts, institutions, and regulations.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"swt-bench":{"id":"swt-bench","name":"SWT-Bench","topic":"Code repair","description":"Software tool-use benchmark for code tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"sysbench":{"id":"sysbench","name":"SysBench","topic":"System prompts","description":"System prompt understanding and adherence benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"t-eval":{"id":"t-eval","name":"T-Eval","topic":"Tool use","description":"Step-by-step evaluation of LLM tool-utilization (instruct, plan, reason, retrieve, understand, review); EN/ZH average.","url":"https://github.com/open-compass/T-Eval","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2312.14033","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tool Use","Text"]},"tabfact":{"id":"tabfact","name":"TabFact","topic":"Table fact verification","description":"Table-based fact verification (MMTab subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"tabmwp":{"id":"tabmwp","name":"TabMWP","topic":"Table math","description":"Tabular math word problems (MMTab subset, image tables).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"tallyqa":{"id":"tallyqa","name":"TallyQA","topic":"Visual counting","description":"TallyQA open-ended counting questions over images (simple and complex).","url":"https://arxiv.org/abs/1810.12440","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Counting"]},"tat-qa":{"id":"tat-qa","name":"TAT-QA","topic":"Table QA","description":"Hybrid tabular and textual financial QA (MMTab subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"tau-voice-airline":{"id":"tau-voice-airline","name":"τ-Voice Airline","topic":"Voice agents","description":"τ-Voice (Sierra): τ-bench airline customer-service tasks over a live speech-to-speech connection; pass^1 (%).","url":"https://sierra.ai/blog/tau-voice","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2603.13686","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Agents"]},"tau-voice-retail":{"id":"tau-voice-retail","name":"τ-Voice Retail","topic":"Voice agents","description":"τ-Voice (Sierra): τ-bench retail customer-service tasks over a live speech-to-speech connection; pass^1 (%).","url":"https://sierra.ai/blog/tau-voice","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2603.13686","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Agents"]},"tau-voice-telecom":{"id":"tau-voice-telecom","name":"τ-Voice Telecom","topic":"Voice agents","description":"τ-Voice (Sierra): τ²-bench telecom tasks over a live speech-to-speech connection; pass^1 (%).","url":"https://sierra.ai/blog/tau-voice","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2603.13686","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Agents"]},"tau1-airline":{"id":"tau1-airline","name":"TAU1-Airline","topic":"Agent tasks (airline)","description":"Tool-augmented agent evaluation in airline scenarios (TAU1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"tau1-retail":{"id":"tau1-retail","name":"TAU1-Retail","topic":"Agent tasks (retail)","description":"Tool-augmented agent evaluation in retail scenarios (TAU1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"tau2-airline":{"id":"tau2-airline","name":"TAU2-Airline","topic":"Agent tasks (airline)","description":"Tool-augmented agent evaluation in airline scenarios (TAU2).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"tau2-airline-verified":{"id":"tau2-airline-verified","name":"TAU2-Bench Airline (Verified)","topic":"Agent tasks","description":"Human-validated revision of the tau2-bench airline domain (amazon-agi/tau2-bench-verified).","url":"https://github.com/amazon-agi/tau2-bench-verified","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"tau2-bench":{"id":"tau2-bench","name":"TAU2-Bench","topic":"Agent tasks","description":"Aggregate tool-augmented agent evaluation across airline, retail, and telecom scenarios (TAU2).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"tau2-retail":{"id":"tau2-retail","name":"TAU2-Retail","topic":"Agent tasks (retail)","description":"Tool-augmented agent evaluation in retail scenarios (TAU2).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"tau2-retail-verified":{"id":"tau2-retail-verified","name":"TAU2-Bench Retail (Verified)","topic":"Agent tasks","description":"Human-validated revision of the tau2-bench retail domain (amazon-agi/tau2-bench-verified).","url":"https://github.com/amazon-agi/tau2-bench-verified","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"tau2-telecom":{"id":"tau2-telecom","name":"TAU2-Telecom","topic":"Agent tasks (telecom)","description":"Tool-augmented agent evaluation in telecom scenarios (TAU2).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"tau3-airline":{"id":"tau3-airline","name":"τ³-Bench Airline","topic":"Agent tasks (airline)","description":"τ³-Bench airline domain: tool-augmented customer-service agent with simulated user.","url":"https://github.com/sierra-research/tau2-bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"tau3-banking":{"id":"tau3-banking","name":"τ³-Bench Banking","topic":"Agent tasks (banking)","description":"τ³-Bench banking domain: tool-augmented customer-service agent with knowledge retrieval and simulated user.","url":"https://github.com/sierra-research/tau2-bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"tau3-banking-voice":{"id":"tau3-banking-voice","name":"τ³-Banking (Voice)","topic":"Voice agents","description":"Sierra τ³ banking knowledge tasks run voice-to-voice; pass@1 (%).","url":"https://sierra.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Agents"]},"tau3-retail":{"id":"tau3-retail","name":"τ³-Bench Retail","topic":"Agent tasks (retail)","description":"τ³-Bench retail domain: tool-augmented customer-service agent with simulated user.","url":"https://github.com/sierra-research/tau2-bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"tau3-telecom":{"id":"tau3-telecom","name":"τ³-Bench Telecom","topic":"Agent tasks (telecom)","description":"τ³-Bench telecom domain: tool-augmented customer-service agent with simulated user (dual-control).","url":"https://github.com/sierra-research/tau2-bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"tax-agent-bench":{"id":"tax-agent-bench","name":"Tax Agent Bench","topic":"Agentic tax research","description":"Vals AI benchmark of agents answering research-grade US corporate tax questions.","url":"https://www.vals.ai/benchmarks/tax_agent_bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Finance","Text"]},"taxeval":{"id":"taxeval","name":"TaxEval","topic":"Finance QA","description":"Tax policy and accounting question answering benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Finance","Knowledge","Text"]},"tempcompass":{"id":"tempcompass","name":"TempCompass","topic":"Temporal reasoning","description":"Temporal reasoning benchmark evaluating understanding of time-related concepts in videos and images.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Temporal","Video","Vision"]},"tense-change":{"id":"tense-change","name":"Tense Change","topic":"Safety / jailbreak","description":"Robustness to adversarial tense-change jailbreak techniques.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"terminal-bench":{"id":"terminal-bench","name":"Terminal-Bench","topic":"Agent terminal tasks","description":"Command-line task completion benchmark for agents.","url":"https://www.tbench.ai/","stars":"637","updated_at":"2025-09-05","leaderboard":"https://www.tbench.ai/leaderboard","github":"https://github.com/laude-institute/terminal-bench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"terminal-bench-2.0":{"id":"terminal-bench-2.0","name":"Terminal-Bench 2.0","topic":"Agent terminal tasks","description":"Second-generation Terminal-Bench leaderboard for end-to-end terminal agents.","url":"https://www.tbench.ai/","stars":"","updated_at":"","leaderboard":"https://www.tbench.ai/leaderboard","github":"https://github.com/laude-institute/terminal-bench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"terminal-bench-2.1":{"id":"terminal-bench-2.1","name":"Terminal-Bench 2.1","topic":"Agent terminal tasks","description":"Updated Terminal-Bench 2.x task set (89 tasks) for end-to-end terminal/command-line agents; successor to Terminal-Bench 2.0.","url":"https://www.tbench.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"terminal-bench-3.0":{"id":"terminal-bench-3.0","name":"Terminal-Bench 3.0","topic":"Agentic terminal tasks","description":"Terminal-Bench 3.0: harder long-horizon command-line agent tasks scored by official verifiers.","url":"https://www.tbench.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"terminal-bench-4.0":{"id":"terminal-bench-4.0","name":"Terminal-Bench 4.0","topic":"Agentic terminal tasks","description":"Terminal-Bench 4.0: complex multi-step professional tasks in a command-line interface.","url":"https://www.tbench.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"terminal-bench-4.0-aa":{"id":"terminal-bench-4.0-aa","name":"Terminal-Bench 4.0-AA","topic":"Agentic terminal coding","description":"Artificial Analysis run of Terminal-Bench 4.0.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"terminal-bench-hard":{"id":"terminal-bench-hard","name":"Terminal-Bench Hard","topic":"Agent terminal tasks","description":"Hard subset of Terminal-Bench command-line agent tasks.","url":"https://www.tbench.ai/","stars":"","updated_at":"2025-09-09","leaderboard":"https://artificialanalysis.ai/evaluations/terminalbench-hard","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"terminal-bench-lite":{"id":"terminal-bench-lite","name":"Terminal-Bench Lite","topic":"Agent terminal tasks","description":"OpenThoughts TBLite: lighter, faster subset-style variant of Terminal-Bench 2.0 for evaluating terminal agents.","url":"https://huggingface.co/datasets/open-thoughts/OpenThoughts-TBLite","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":""},"terminal-bench-science":{"id":"terminal-bench-science","name":"Terminal-Bench-Science","topic":"Agentic scientific research","description":"Terminal-Bench-Science 0.1: agentic scientific research tasks in terminal environments.","url":"https://www.tbench.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","STEM","Text"]},"terminal-bench-terminus":{"id":"terminal-bench-terminus","name":"Terminal-Bench Terminus","topic":"Agent terminal tasks","description":"Terminal-Bench Terminus track assessing end-to-end terminal tool use.","url":"https://www.tbench.ai/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/laude-institute/terminal-bench","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"textquests":{"id":"textquests","name":"TextQuests","topic":"Text-based video games","description":"Text-based video game benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Games","Reasoning","Text"]},"textquests-harm":{"id":"textquests-harm","name":"TextQuests Harm","topic":"Harmful propensities","description":"Harmfulness evaluation on TextQuests scenarios.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":false,"classification":["Safety","Harm","Text"]},"textvqa":{"id":"textvqa","name":"TextVQA","topic":"Text-based VQA","description":"Visual question answering that requires reading text in images.","url":"https://textvqa.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","OCR","Vision"]},"theoremqa":{"id":"theoremqa","name":"TheoremQA","topic":"Theorem-driven STEM QA","description":"Theorem-driven question answering in math, physics, EE/CS and finance.","url":"https://github.com/TIGER-AI-Lab/TheoremQA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2305.12524","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"tiif-bench-long":{"id":"tiif-bench-long","name":"TIIF-Bench Long","topic":"Text-to-image","description":"TIIF-Bench long prompt score for text-to-image generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"tiif-bench-short":{"id":"tiif-bench-short","name":"TIIF-Bench Short","topic":"Text-to-image","description":"TIIF-Bench short prompt score for text-to-image generation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Generation","Multimodal","Vision"]},"timelens-activitynet":{"id":"timelens-activitynet","name":"ActivityNet-TimeLens","topic":"Video temporal grounding","description":"TimeLens-Bench re-annotated ActivityNet Captions split for video temporal grounding (mIoU).","url":"https://arxiv.org/abs/2512.14698","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Grounding","Vision"]},"timelens-charades":{"id":"timelens-charades","name":"Charades-TimeLens","topic":"Video temporal grounding","description":"TimeLens-Bench re-annotated Charades-STA split for video temporal grounding (mIoU).","url":"https://arxiv.org/abs/2512.14698","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Grounding","Vision"]},"timelens-qvhighlights":{"id":"timelens-qvhighlights","name":"QVHighlights-TimeLens","topic":"Video temporal grounding","description":"TimeLens-Bench re-annotated QVHighlights split for video temporal grounding (mIoU).","url":"https://arxiv.org/abs/2512.14698","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Grounding","Vision"]},"tir-bench":{"id":"tir-bench","name":"TIR-Bench","topic":"Tool-integrated reasoning","description":"Benchmark for tool-integrated reasoning with visual models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Tools","Vision"]},"tldr9-plus":{"id":"tldr9-plus","name":"TLDR9+","topic":"Summarization","description":"Long-form summarization benchmark with nine-domain TLDR prompts plus extended variations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Summarization","Writing","Text"]},"tomato":{"id":"tomato","name":"TOMATO","topic":"Temporal understanding","description":"Temporal ordering and motion analysis benchmark (TOMATO).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Temporal","Video","Reasoning","Vision"]},"tomg-bench":{"id":"tomg-bench","name":"TOMG-Bench","topic":"Molecule generation","description":"Text-based open molecule generation benchmark (editing, optimization, customized generation).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2412.14642","modality":"text","metric":"percent","higher_is_better":true,"classification":["Chemistry","Science","Text"]},"tool-decathlon":{"id":"tool-decathlon","name":"Tool-Decathlon","topic":"Agent tool-use","description":"Composite tool-use suite measuring multi-domain tool invocation success (Pass@1).","url":"","stars":"","updated_at":"2025-12-01","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"toolace":{"id":"toolace","name":"ToolACE","topic":"Tool calling","description":"ToolACE function-calling evaluation, posed as decisions.","url":"https://huggingface.co/datasets/Team-ACE/ToolACE","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tool Use","Text"]},"toolathlon":{"id":"toolathlon","name":"Toolathlon","topic":"Agentic software tasks","description":"Long-horizon, real-world software tool-use tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Reasoning","Text"]},"toolathlon-verified":{"id":"toolathlon-verified","name":"Toolathlon Verified","topic":"Tool use","description":"Verified version of Toolathlon (The Tool Decathlon), run via the official evaluation service; pass@1 averaged over 3 runs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"toolhop":{"id":"toolhop","name":"ToolHop","topic":"Multi-hop tool use","description":"Query-driven benchmark of 995 multi-hop tool-use queries with 3,912 locally executable tools; answer accuracy.","url":"https://arxiv.org/abs/2501.02506","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tool Use","Text"]},"toolret-decision":{"id":"toolret-decision","name":"ToolRet (decision)","topic":"Tool retrieval","description":"Rate usefulness of 32 retrieved tools per request; nDCG@10. Jev Decision Index adaptation posed as typed decisions; not comparable to the generative benchmark.","url":"https://huggingface.co/datasets/mangopy/ToolRet-Queries","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"ndcg@10","higher_is_better":true,"classification":["Retrieval","Tool Use","Text"]},"toolsandbox":{"id":"toolsandbox","name":"ToolSandbox","topic":"Tool use","description":"Apple's stateful, conversational, interactive tool-use benchmark with simulated user.","url":"https://github.com/apple/ToolSandbox","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"topiocqa":{"id":"topiocqa","name":"TopiOCQA","topic":"Conversational QA","description":"Topic-switching open-domain conversational QA (ChatRAG-Bench subset).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["QA","Text"]},"totaltext-f1":{"id":"totaltext-f1","name":"TotalText (F1)","topic":"OCR","description":"TotalText text grounding and recognition, F1 mean.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","OCR"]},"toxigen":{"id":"toxigen","name":"ToxiGen","topic":"Toxicity","description":"Toxic language generation and detection benchmark.","url":"https://huggingface.co/datasets/toxigen/toxigen-data","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/microsoft/TOXIGEN","paper":"https://arxiv.org/abs/2203.09509","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Toxicity","Bias","Text"]},"treebench":{"id":"treebench","name":"TreeBench","topic":"Reasoning with tree structures","description":"Evaluates hierarchical/tree-structured reasoning and planning capabilities in LLMs/VLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2507.07999","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"triqa":{"id":"triqa","name":"TriQA","topic":"Knowledge QA","description":"Triadic question answering benchmark evaluating world knowledge and reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"triviaqa":{"id":"triviaqa","name":"TriviaQA","topic":"Open-domain QA","description":"Open-domain question answering benchmark built from trivia and web evidence.","url":"https://nlp.cs.washington.edu/triviaqa/","stars":"","updated_at":"2025-07-29","leaderboard":"","github":"https://github.com/mandarjoshi90/triviaqa","paper":"https://arxiv.org/abs/1705.03551","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"triviaqa-wiki":{"id":"triviaqa-wiki","name":"TriviaQA-Wiki","topic":"Open-domain QA","description":"TriviaQA subset answering using Wikipedia evidence.","url":"http://nlp.cs.washington.edu/triviaqa/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Text"]},"trustllm":{"id":"trustllm","name":"TrustLLM","topic":"Safety / reliability","description":"TrustLLM benchmark for trustworthiness and safety behaviors.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"truthfulqa":{"id":"truthfulqa","name":"TruthfulQA","topic":"Truthfulness / hallucination","description":"Measures whether a model imitates human falsehoods (truthfulness).","url":"https://github.com/sylinrl/TruthfulQA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2109.07958","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Hallucination","Text"]},"truthfulqa-binary":{"id":"truthfulqa-binary","name":"TruthfulQA (binary)","topic":"Truthfulness","description":"Binary-choice variant of TruthfulQA.","url":"https://github.com/sylinrl/TruthfulQA","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Truthfulness","Text"]},"truthfulqa-de":{"id":"truthfulqa-de","name":"TruthfulQA (DE)","topic":"Truthfulness / hallucination (German)","description":"German translation of the TruthfulQA benchmark.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"norm_prob_mass","higher_is_better":true,"classification":["Safety","Hallucination","Text","German"]},"truthqa":{"id":"truthqa","name":"TruthQA","topic":"Truthfulness","description":"TruthQA benchmark assessing truthfulness on web-derived claims.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text","Web"]},"truthqa-multilingual":{"id":"truthqa-multilingual","name":"TruthfulQA Multilingual","topic":"Truthfulness (multilingual)","description":"Multilingual variant of TruthfulQA across multiple languages.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"turkishmmlu":{"id":"turkishmmlu","name":"TurkishMMLU","topic":"Turkish knowledge","description":"Native Turkish multiple-choice QA benchmark from Turkish curriculum exams.","url":"https://huggingface.co/datasets/AYueksel/TurkishMMLU","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2407.12402","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multilingual","Text"]},"tvbench":{"id":"tvbench","name":"TVBench","topic":"TV comprehension","description":"Benchmark for TV show video comprehension and QA.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","QA","Vision"]},"tydiqa":{"id":"tydiqa","name":"TydiQA","topic":"Cross-lingual QA","description":"Typologically diverse QA across languages.","url":"https://github.com/google-research-datasets/tydiqa","stars":"313","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","QA","Reasoning","Text"]},"typed-decision":{"id":"typed-decision","name":"Typed Decision","topic":"Typed decisions","description":"Typed-decision evaluation set used in Intern-Decision model card (Laya Typed Decisions).","url":"https://huggingface.co/internlm/Intern-Decision-4B","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Decision","Text"]},"u-artifacts":{"id":"u-artifacts","name":"U-Artifacts","topic":"Agentic coding artifacts","description":"Benchmark focusing on generated code artifacts quality.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Agents","Text"]},"ugc-videocap":{"id":"ugc-videocap","name":"UGC-VideoCap","topic":"Video captioning","description":"Short-form user-generated video captioning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"ui-vision":{"id":"ui-vision","name":"UI-Vision","topic":"GUI grounding","description":"Desktop GUI grounding benchmark (basic, functional, spatial) for computer-use agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2503.15661","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"unimernet":{"id":"unimernet","name":"UniMERNet","topic":"Formula recognition","description":"UniMER-Test benchmark for universal mathematical expression recognition from images.","url":"https://github.com/opendatalab/UniMERNet","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Vision","Math"]},"uro-bench-pro-oral":{"id":"uro-bench-pro-oral","name":"URO-Bench Pro: Oral Conversation","topic":"Spoken dialogue","description":"URO-Bench pro track, oral-conversation dimension, speech-to-speech (%).","url":"https://github.com/Ruiqi-Yan/URO-Bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2502.17810","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Chat"]},"uro-bench-pro-reasoning":{"id":"uro-bench-pro-reasoning","name":"URO-Bench Pro: Reasoning","topic":"Spoken dialogue","description":"URO-Bench pro track, reasoning dimension, speech-to-speech (%).","url":"https://github.com/Ruiqi-Yan/URO-Bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2502.17810","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Reasoning"]},"uro-bench-pro-understanding":{"id":"uro-bench-pro-understanding","name":"URO-Bench Pro: Understanding","topic":"Spoken dialogue","description":"URO-Bench pro track, understanding dimension, speech-to-speech (%).","url":"https://github.com/Ruiqi-Yan/URO-Bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2502.17810","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice"]},"usaco":{"id":"usaco","name":"USACO","topic":"Code generation (olympiad)","description":"USA Computing Olympiad problems benchmark (307 problems); pass@1.","url":"https://princeton-nlp.github.io/USACOBench/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2404.10952","modality":"text","metric":"percent","higher_is_better":true,"classification":["Code","Text"]},"usamo-2026":{"id":"usamo-2026","name":"USAMO 2026","topic":"Math olympiad (proofs)","description":"2026 USA Mathematical Olympiad (6 proof problems), graded with MathArena methodology.","url":"https://matharena.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","Text"]},"v-star":{"id":"v-star","name":"V*","topic":"Multimodal reasoning","description":"V* benchmark accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"v-star-with-tools":{"id":"v-star-with-tools","name":"V* (With Tools)","topic":"Multimodal reasoning","description":"V* visual search benchmark evaluated with a Python code-interpreter tool.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"vabench-point":{"id":"vabench-point","name":"VABench-Point","topic":"Embodied reasoning","description":"VABench visual-action pointing subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"vabench-visual-trace":{"id":"vabench-visual-trace","name":"VABench-VisualTrace","topic":"Embodied reasoning","description":"VABench visual-trace (trajectory) subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"vals-index":{"id":"vals-index","name":"Vals Index","topic":"Knowledge work","description":"Vals AI composite index of real-world professional/enterprise tasks (finance, legal, coding, etc.).","url":"https://www.vals.ai/","stars":"","updated_at":"","leaderboard":"https://www.vals.ai/","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Knowledge Work","Text"]},"vast-stance":{"id":"vast-stance","name":"VAST","topic":"Stance detection","description":"VAST zero-shot stance detection; macro-F1 (Jev Decision Index suite, posed as typed decisions).","url":"https://github.com/emilyallaway/zero-shot-stance","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/multimodalart/jev-decision-index","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Classification","Text"]},"vcr-en-easy":{"id":"vcr-en-easy","name":"VCR (EN, easy)","topic":"Visual caption restoration","description":"Visual Caption Restoration, English easy split; exact-match accuracy of restoring partially occluded text.","url":"https://github.com/tianyu-z/VCR","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2406.06462","modality":"visual","metric":"percent","higher_is_better":true,"classification":["OCR","Multimodal","Vision"]},"vcrbench":{"id":"vcrbench","name":"VCRBench","topic":"Visual commonsense reasoning","description":"Visual commonsense reasoning benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Reasoning","Multimodal"]},"vct":{"id":"vct","name":"VCT","topic":"Virology capability (protocol troubleshooting)","description":"Virology Capabilities Test: a benchmark that measures an LLM's ability to troubleshoot complex virology laboratory protocols.","url":"","stars":"","updated_at":"","leaderboard":"https://www.virologytest.ai/","github":"","paper":"https://arxiv.org/abs/2504.16137","modality":"text","metric":"percent","higher_is_better":true,"classification":["Biosecurity","Science","Reasoning","Text"]},"vending-bench-2":{"id":"vending-bench-2","name":"Vending-Bench 2","topic":"Long-horizon agentic tasks","description":"Long-horizon agentic task benchmark evaluating sustained goal completion.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"net_worth_mean","higher_is_better":true,"classification":["Agents","Text"]},"vibe-android":{"id":"vibe-android","name":"Vibe Android","topic":"Vibe evaluation (Android)","description":"Vibe evaluation on Android tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Mobile","Text"]},"vibe-average":{"id":"vibe-average","name":"Vibe Average","topic":"Vibe evaluation","description":"Aggregate Vibe evaluation score.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Meta","Text"]},"vibe-backend":{"id":"vibe-backend","name":"Vibe Backend","topic":"Vibe evaluation (backend)","description":"Vibe evaluation on backend tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"vibe-code-bench":{"id":"vibe-code-bench","name":"Vibe Code Bench","topic":"App generation","description":"Vals AI Vibe Code Bench: building complete web applications from natural-language specs.","url":"https://www.vals.ai/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Code","Text"]},"vibe-code-bench-v1.1":{"id":"vibe-code-bench-v1.1","name":"Vibe Code Bench v1.1","topic":"Application building","description":"Vals AI Vibe Code Bench v1.1: build working web applications from natural-language specs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Coding"},"vibe-eval-reka":{"id":"vibe-eval-reka","name":"Vibe-Eval (Reka)","topic":"Safety / response style","description":"Reka's Vibe-Eval measuring subjective style and helpfulness preferences.","url":"","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"vibe-ios":{"id":"vibe-ios","name":"Vibe iOS","topic":"Vibe evaluation (iOS)","description":"Vibe evaluation on iOS tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Mobile","Text"]},"vibe-simulation":{"id":"vibe-simulation","name":"Vibe Simulation","topic":"Vibe evaluation (simulation)","description":"Vibe evaluation on simulation tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Simulation","Text"]},"vibe-web":{"id":"vibe-web","name":"Vibe Web","topic":"Vibe evaluation (web)","description":"Vibe evaluation on web tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Web","Text"]},"vibeval":{"id":"vibeval","name":"VibeEval","topic":"Aesthetic/visual quality","description":"VLM aesthetic evaluation with GPT scores.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"video-mme":{"id":"video-mme","name":"Video-MME","topic":"Video understanding (multimodal)","description":"Multimodal evaluation of video understanding and reasoning.","url":"https://video-mme.github.io/home_page.html","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Video","Vision"]},"videoevalpro":{"id":"videoevalpro","name":"VideoEval-Pro","topic":"Video understanding","description":"Long video understanding benchmark with open-ended short-answer questions.","url":"https://arxiv.org/abs/2505.14640","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":""},"videohallucer":{"id":"videohallucer","name":"VideoHallucer","topic":"Video hallucination","description":"Benchmark for detecting intrinsic and extrinsic hallucinations in video-language models.","url":"https://videohallucer.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Hallucination"]},"videoholmes":{"id":"videoholmes","name":"VideoHolmes","topic":"Video QA","description":"Video question answering benchmark focused on detective-style clues.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","QA","Reasoning","Vision"]},"videomathqa":{"id":"videomathqa","name":"VideoMathQA","topic":"Video math reasoning","description":"Benchmark for mathematical reasoning grounded in video content.","url":"https://arxiv.org/abs/2506.05349","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Math","Reasoning","Vision"]},"videomme":{"id":"videomme","name":"VideoMME","topic":"Multimodal video evaluation","description":"Video multimodal evaluation suite (VideoMME).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"videomme-nosub":{"id":"videomme-nosub","name":"VideoMME (w/o sub)","topic":"Video understanding","description":"Video understanding benchmark without subtitles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"videomme-sub":{"id":"videomme-sub","name":"VideoMME (w/sub)","topic":"Video understanding","description":"Video understanding benchmark with subtitles.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal","Vision"]},"videomme-v2":{"id":"videomme-v2","name":"Video-MME v2","topic":"Video understanding","description":"Second version of the Video-MME video multimodal evaluation suite.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":"Multimodal"},"videommmu":{"id":"videommmu","name":"VideoMMMU","topic":"Multimodal video understanding","description":"Video-based extension of MMMU evaluating temporal multimodal reasoning and perception across disciplines.","url":"","stars":"","updated_at":"2025-09-09","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Video","Vision"]},"videoreasonbench":{"id":"videoreasonbench","name":"VideoReasonBench","topic":"Video reasoning","description":"Video reasoning benchmark assessing temporal and causal understanding.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Reasoning","Vision"]},"videosimpleqa":{"id":"videosimpleqa","name":"VideoSimpleQA","topic":"Video QA","description":"Simple question answering over short videos.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","QA","Vision"]},"viewspatial-bench":{"id":"viewspatial-bench","name":"ViewSpatial-Bench","topic":"Spatial reasoning","description":"Multi-perspective spatial localization benchmark (camera vs. human viewpoints).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"visdrone-det-f1":{"id":"visdrone-det-f1","name":"VisDrone Detection (F1)","topic":"Object detection","description":"VisDrone object detection with an MLLM; F1 averaged over IoU 0.5:0.95.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"visdrone-pointing":{"id":"visdrone-pointing","name":"VisDrone Pointing","topic":"Object pointing","description":"VisDrone object pointing accuracy.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Grounding"]},"visfactor":{"id":"visfactor","name":"VisFactor","topic":"Visual perception","description":"Benchmark of fundamental visual-cognition factors adapted from human psychometric perception tests.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal"]},"vision2web":{"id":"vision2web","name":"Vision2Web","topic":"Coding (UI)","description":"Visual website generation benchmark: build working websites from visual designs/prototypes, judged on rendered result.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Code","UI","Multimodal"]},"vispeak":{"id":"vispeak","name":"ViSpeak","topic":"Video dialogue","description":"Video-grounded dialogue and description benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Dialogue","Vision"]},"visualmrc":{"id":"visualmrc","name":"VisualMRC","topic":"Document visual machine reading comprehension","description":"Abstractive QA over web-page screenshots; scored with CIDEr.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/nttmdlab-nlp/VisualMRC","paper":"https://arxiv.org/abs/2101.11272","modality":"visual","metric":"CIDEr","higher_is_better":true,"classification":["Document","QA","Vision"]},"visualprobe":{"id":"visualprobe","name":"VisualProbe","topic":"Visual search / fine-grained perception","description":"High-resolution visual search benchmark requiring iterative zoom/crop tool use (introduced with Mini-o3).","url":"https://arxiv.org/abs/2509.07969","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2509.07969","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Agents"]},"visualpuzzle":{"id":"visualpuzzle","name":"VisualPuzzle","topic":"Visual reasoning","description":"Visual puzzle solving benchmark evaluating reasoning and pattern recognition capabilities.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"visualwebbench-avg":{"id":"visualwebbench-avg","name":"VisualWebBench","topic":"Web UI understanding","description":"Average accuracy on VisualWebBench.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/VisualWebBench/VisualWebBench","paper":"https://arxiv.org/abs/2404.05955","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Multimodal","Vision"]},"visulogic":{"id":"visulogic","name":"VisuLogic","topic":"Visual logical reasoning","description":"Logical reasoning and compositionality benchmark for visual-language models.","url":"https://visulogic-benchmark.github.io/VisuLogic/","stars":"","updated_at":"","leaderboard":"https://visulogic-benchmark.github.io/VisuLogic/","github":"https://github.com/VisuLogic-Benchmark/VisuLogic-Eval","paper":"https://arxiv.org/abs/2504.15279","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Reasoning","Vision"]},"vitabench":{"id":"vitabench","name":"VitaBench","topic":"Industry QA","description":"Industry-focused benchmark evaluating domain QA performance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Industry","Knowledge","QA","Text"]},"vitabench-2.0":{"id":"vitabench-2.0","name":"VitaBench 2.0","topic":"Agentic real-world tasks","description":"VitaBench 2.0: interactive agent tasks in real-world service scenarios (v2.0, not comparable to v1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"vizwiz":{"id":"vizwiz","name":"VizWiz-VQA","topic":"Accessibility VQA","description":"Visual question answering on photos taken by blind people (VQA score).","url":"https://vizwiz.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1802.08218","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"vl-rewardbench":{"id":"vl-rewardbench","name":"VL-RewardBench","topic":"Reward modeling (VL)","description":"Reward alignment benchmark for VLMs.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Alignment","Multimodal","Vision"]},"vlabench":{"id":"vlabench","name":"VLABench","topic":"Embodied reasoning","description":"Language-conditioned robot manipulation reasoning benchmark (VLM track).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Embodied","Vision"]},"vlad":{"id":"vlad","name":"VLAD","topic":"Driving scene QA","description":"Vision-language autonomous-driving QA benchmark (as reported in the Qwen-Drive-1.0 evaluation).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Autonomous Driving","Multimodal","QA","Vision"]},"vlms-are-biased":{"id":"vlms-are-biased","name":"VLMs are Biased","topic":"Multimodal bias","description":"Evaluates whether VLMs truly 'see' vs. relying on memorized knowledge; measures bias toward non-visual priors.","url":"https://vlmsarebiased.github.io/","stars":"90","updated_at":"","leaderboard":"https://vlmsarebiased.github.io/","github":"https://github.com/anvo25/vlms-are-biased","paper":"https://arxiv.org/abs/2505.23941","data":"https://huggingface.co/datasets/anvo25/vlms-are-biased","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Bias","Multimodal","Vision"]},"vlms-are-blind":{"id":"vlms-are-blind","name":"VLMs are Blind","topic":"Visual grounding robustness","description":"Evaluates failure modes of VLMs in grounding and perception tasks.","url":"https://vlmsareblind.github.io/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/anguyen8/vision-llms-are-blind","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Grounding","Multimodal","Vision"]},"vlmsarebiased":{"id":"vlmsarebiased","name":"VLMsAreBiased","topic":"Multimodal bias","description":"Benchmark evaluating biases in vision-language models.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Bias","Multimodal","Vision"]},"vlmsareblind":{"id":"vlmsareblind","name":"VLMsAreBlind","topic":"Multimodal robustness","description":"Benchmark probing robustness of vision-language models to visual perturbations.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Robustness","Vision"]},"voicebench-advbench":{"id":"voicebench-advbench","name":"VoiceBench AdvBench","topic":"VoiceBench","description":"VoiceBench adversarial safety evaluation.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Safety","Audio"]},"voicebench-alpacaeval":{"id":"voicebench-alpacaeval","name":"VoiceBench AlpacaEval","topic":"VoiceBench","description":"VoiceBench evaluation on AlpacaEval instructions.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Instruction Following","Audio"]},"voicebench-bbh":{"id":"voicebench-bbh","name":"VoiceBench BBH","topic":"VoiceBench","description":"VoiceBench evaluation on Big-Bench Hard prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Reasoning","Audio"]},"voicebench-commoneval":{"id":"voicebench-commoneval","name":"VoiceBench CommonEval","topic":"VoiceBench","description":"VoiceBench evaluation on CommonEval.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","QA","Audio"]},"voicebench-ifeval":{"id":"voicebench-ifeval","name":"VoiceBench IFEval","topic":"VoiceBench","description":"VoiceBench instruction-following evaluation (IFEval).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Instruction Following","Audio"]},"voicebench-ja-elyza":{"id":"voicebench-ja-elyza","name":"VoiceBench-ja Elyza","topic":"Spoken QA (Japanese)","description":"ELYZA-tasks subset of VoiceBench-ja (spoken input), GPT-4o judge score 0-5.","url":"https://huggingface.co/datasets/sbintuitions/voicebench-ja","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"score","higher_is_better":true,"classification":["Audio","Speech","Japanese"]},"voicebench-ja-m-ifeval":{"id":"voicebench-ja-m-ifeval","name":"VoiceBench-ja M-IFEval","topic":"Spoken instruction following (Japanese)","description":"M-IFEval subset of VoiceBench-ja (spoken input), scored 0-1.","url":"https://huggingface.co/datasets/sbintuitions/voicebench-ja","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"score","higher_is_better":true,"classification":["Audio","Speech","Instruction Following","Japanese"]},"voicebench-ja-spoken-elyza":{"id":"voicebench-ja-spoken-elyza","name":"VoiceBench-ja spoken_Elyza","topic":"Spoken QA (Japanese)","description":"spoken_Elyza subset of VoiceBench-ja, GPT-4o judge score 0-5.","url":"https://huggingface.co/datasets/sbintuitions/voicebench-ja","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"score","higher_is_better":true,"classification":["Audio","Speech","Japanese"]},"voicebench-mau-v05-15-25":{"id":"voicebench-mau-v05-15-25","name":"MMAU v05.15.25","topic":"Audio reasoning","description":"Audio reasoning benchmark MMAU v05.15.25.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Reasoning","Audio"]},"voicebench-mmsu":{"id":"voicebench-mmsu","name":"VoiceBench MMSU","topic":"VoiceBench","description":"VoiceBench MMSU benchmark (voice modality).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","STEM","Audio"]},"voicebench-mmsu-audio":{"id":"voicebench-mmsu-audio","name":"VoiceBench MMSU (Audio)","topic":"Audio reasoning","description":"Audio reasoning MMSU results.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Reasoning","Audio"]},"voicebench-openbookqa":{"id":"voicebench-openbookqa","name":"VoiceBench OpenBookQA","topic":"VoiceBench","description":"VoiceBench results on OpenBookQA prompts.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","Knowledge","QA","Audio"]},"voicebench-sd-qa":{"id":"voicebench-sd-qa","name":"VoiceBench SD-QA","topic":"VoiceBench","description":"VoiceBench Spoken Dialogue QA results.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","QA","Audio"]},"voicebench-wildvoice":{"id":"voicebench-wildvoice","name":"VoiceBench WildVoice","topic":"VoiceBench","description":"VoiceBench evaluation on WildVoice dataset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Voice","QA","Audio"]},"vpct":{"id":"vpct","name":"VPCT","topic":"Multimodal reasoning","description":"Visual perception and comprehension test.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","Vision"]},"vqa-rad":{"id":"vqa-rad","name":"VQA-RAD","topic":"Medical VQA","description":"Radiology visual question answering (closed-question accuracy).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Medical","Vision"]},"vqav2":{"id":"vqav2","name":"VQAv2","topic":"Visual question answering","description":"Standard Visual Question Answering v2 benchmark on natural images.","url":"https://visualqa.org/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Multimodal","QA","Vision"]},"vsi-bench":{"id":"vsi-bench","name":"VSI-Bench","topic":"Spatial intelligence","description":"Visual spatial intelligence benchmark covering 3D reasoning and spatial inference tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Spatial","Vision"]},"waymoqa":{"id":"waymoqa","name":"WaymoQA","topic":"Driving scene QA","description":"Driving-scene multiple-choice QA benchmark built on Waymo data; overall accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Autonomous Driving","Multimodal","QA","Vision"]},"waymoqa-safety":{"id":"waymoqa-safety","name":"WaymoQA (Safety)","topic":"Driving scene QA","description":"Safety-critical subset of WaymoQA; accuracy.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Autonomous Driving","Multimodal","QA","Vision","Safety"]},"webarena":{"id":"webarena","name":"WebArena","topic":"Web agents","description":"Realistic self-hosted web environments (shopping, forums, GitLab, CMS, maps) for autonomous web agents; task success rate.","url":"https://webarena.dev/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","Web"]},"webarena-verified":{"id":"webarena-verified","name":"WebArena-Verified","topic":"Web automation","description":"Verified version of the WebArena browser-use benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Web","Agents","Multimodal"]},"webclick":{"id":"webclick","name":"WebClick","topic":"GUI agents","description":"Task success on the WebClick UI agent benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Multimodal","Vision"]},"webdev-arena":{"id":"webdev-arena","name":"WebDev Arena","topic":"Web development agents","description":"Arena evaluation for autonomous web development agents.","url":"","stars":"","updated_at":"","leaderboard":"https://lmarena.ai/leaderboard/webdev","github":"","paper":"","modality":"text","metric":"elo","higher_is_better":true,"classification":["Agents","Code"]},"webquest-multiqa":{"id":"webquest-multiqa","name":"WebQuest-MultiQA","topic":"Web agents","description":"Multi-question web search and interaction tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Web","Text"]},"webquest-singleqa":{"id":"webquest-singleqa","name":"WebQuest-SingleQA","topic":"Web agents","description":"Single-question web search and interaction tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Web","Text"]},"websrc":{"id":"websrc","name":"WebSrc","topic":"Web QA","description":"Webpage question answering (SQuAD F1).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["GUI","Web","Multimodal","Vision"]},"webtailbench":{"id":"webtailbench","name":"WebTailBench","topic":"Web agents","description":"Microsoft long-tail web task benchmark (shopping, travel, comparison and other real-world tasks) introduced with Fara-7B.","url":"https://huggingface.co/datasets/microsoft/WebTailBench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":""},"webtestbench":{"id":"webtestbench","name":"WebTestBench","topic":"Web testing agents","description":"Benchmark of web-app testing agents that detect defects against a checklist in live web applications.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"webvoyager":{"id":"webvoyager","name":"WebVoyager","topic":"Web agents","description":"Web navigation and interaction tasks for LLM agents.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Web","Text"]},"webvoyager2":{"id":"webvoyager2","name":"WebVoyager2","topic":"Web agents","description":"Web navigation and interaction tasks for LLM agents (v2).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Web","Text"]},"webwalker-qa":{"id":"webwalker-qa","name":"WebWalkerQA","topic":"Web agents","description":"WebWalker tasks evaluating autonomous browsing question answering performance.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"avg@3","higher_is_better":true,"classification":["Agents","Web","Text"]},"wemath":{"id":"wemath","name":"WeMath","topic":"Math reasoning","description":"Math reasoning benchmark spanning diverse curricula and difficulty levels.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Math","Reasoning","STEM","Text"]},"weref-f1":{"id":"weref-f1","name":"WeRef (F1)","topic":"Referring expressions","description":"Tencent WeRef referring object detection benchmark, F1 mean.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"f1","higher_is_better":true,"classification":["Vision","Multimodal","Referring"]},"weref-rejection":{"id":"weref-rejection","name":"WeRef (Rejection)","topic":"Referring expressions","description":"Rejection rate on non-existent referents in WeRef.","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Multimodal","Referring","Hallucination"]},"werefmat-category-mad":{"id":"werefmat-category-mad","name":"WeRefMat (Category) Matting (MAD)","topic":"Image matting","description":"WeRefMat (Category): category-prompted matting; mean absolute difference x10^3 (lower is better).","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"mad","higher_is_better":false,"classification":["Vision","Multimodal","Perception"]},"werefmat-referring-mad":{"id":"werefmat-referring-mad","name":"WeRefMat (Referring) Matting (MAD)","topic":"Image matting","description":"WeRefMat (Referring): referring-expression matting; mean absolute difference x10^3 (lower is better).","url":"https://github.com/Tencent/WeVisOne-VL","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2610.31329","modality":"visual","metric":"mad","higher_is_better":false,"classification":["Vision","Multimodal","Perception"]},"werewolf":{"id":"werewolf","name":"Werewolf","topic":"Social intelligence","description":"Social deduction multi-agent gameplay benchmark.","url":"https://huggingface.co/spaces/Foaster/Werewolf_benchmark","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"wgo-bench":{"id":"wgo-bench","name":"WGO Bench","topic":"Embodied manipulation","description":"WGO Bench: end-to-end robot manipulation event understanding (Elorian-reported).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"video","metric":"percent","higher_is_better":true,"classification":["Video","Multimodal"]},"when2call":{"id":"when2call","name":"When2Call","topic":"Tool calling","description":"NVIDIA's When2Call multiple-choice test set: choose whether to answer directly, call a tool, ask for more information, or say it cannot answer.","url":"https://huggingface.co/datasets/nvidia/When2Call","stars":"","updated_at":"","leaderboard":"https://artificialanalysis.ai/evaluations/when2call","github":"","paper":"https://arxiv.org/abs/2504.18851","modality":"text","metric":"percent","higher_is_better":true,"classification":["Tool Use","Agents","Text"]},"where2place":{"id":"where2place","name":"Where2Place","topic":"Embodied spatial understanding","description":"Spatial free-space pointing benchmark: predict points for placement locations described in language.","url":"https://arxiv.org/abs/2406.10721","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2406.10721","modality":"visual","metric":"percent","higher_is_better":true,"classification":""},"wide-search":{"id":"wide-search","name":"WideSearch","topic":"Web search","description":"Wide web search and QA benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Search","Web","Text"]},"wikiwebfacts":{"id":"wikiwebfacts","name":"WikiWebFacts","topic":"Russian factual knowledge","description":"Yandex 5-shot benchmark of factual knowledge in Russian.","url":"https://huggingface.co/datasets/yandex/WikiWebFacts","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Knowledge","Text"]},"wild-jailbreak":{"id":"wild-jailbreak","name":"Wild-Jailbreak","topic":"Safety / jailbreak","description":"Adversarial jailbreak benchmark evaluating refusal robustness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Adversarial","Text"]},"wildbench-v2":{"id":"wildbench-v2","name":"WildBench V2","topic":"Instruction following","description":"WildBench V2 human preference benchmark for instruction following and helpfulness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Instruction Following","Text"]},"wildclawbench":{"id":"wildclawbench","name":"WildClawBench","topic":"Agentic tasks","description":"Agentic benchmark of in-the-wild tasks run in OpenClaw-style agent scaffolds.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":"Agents"},"wildguardtest":{"id":"wildguardtest","name":"WildGuardTest","topic":"Safety","description":"WildGuardTest safety benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"wildspeech-bench":{"id":"wildspeech-bench","name":"WildSpeech-Bench","topic":"Spoken dialogue","description":"WildSpeech-Bench: end-to-end speech-in/speech-out evaluation on real-world spoken queries (%).","url":"https://github.com/Tencent/WildSpeech-Bench","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"audio","metric":"percent","higher_is_better":true,"classification":["Audio","Voice","Chat"]},"wildvision":{"id":"wildvision","name":"WildVision-Bench","topic":"Multimodal chat","description":"In-the-wild multimodal chat benchmark judged against reference model (WildVision Arena).","url":"https://huggingface.co/spaces/WildVision/vision-arena","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Chat","Vision"]},"windows-agent-arena":{"id":"windows-agent-arena","name":"Windows Agent Arena","topic":"GUI agents","description":"Windows OS computer-use agent benchmark (WAA), task success rate.","url":"https://microsoft.github.io/WindowsAgentArena/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Agents","GUI","Vision"]},"winogender":{"id":"winogender","name":"Winogender","topic":"Gender bias (coreference)","description":"Coreference resolution dataset for measuring gender bias.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/rudinger/winogender-schemas","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"winogrande":{"id":"winogrande","name":"WinoGrande","topic":"Coreference reasoning","description":"Large-scale adversarial Winograd Schema-style pronoun resolution.","url":"https://github.com/allenai/winogrande","stars":"99","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"winogrande-de":{"id":"winogrande-de","name":"WinoGrande (DE)","topic":"Coreference reasoning (German)","description":"German translation of the WinoGrande pronoun resolution benchmark.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"norm_log_acc","higher_is_better":true,"classification":["Reasoning","Text","German"]},"wmdp-bio":{"id":"wmdp-bio","name":"WMDP Bio","topic":"Biosecurity knowledge","description":"Weapons of Mass Destruction Proxy benchmark for biosecurity, measuring hazardous biological knowledge without info hazards.","url":"https://www.wmdp.ai/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/centerforaisafety/wmdp","paper":"https://arxiv.org/abs/2403.03218","modality":"text","metric":"percent","higher_is_better":false,"classification":["Knowledge","QA","Reasoning","Text"]},"wmdp-chem":{"id":"wmdp-chem","name":"WMDP Chem","topic":"Chemical security knowledge","description":"WMDP benchmark for chemical security, evaluating knowledge relevant to chemical weapons development.","url":"https://www.wmdp.ai/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/centerforaisafety/wmdp","paper":"https://arxiv.org/abs/2403.03218","modality":"text","metric":"percent","higher_is_better":false,"classification":["Knowledge","QA","Text"]},"wmdp-cyber":{"id":"wmdp-cyber","name":"WMDP Cyber","topic":"Cybersecurity knowledge","description":"WMDP benchmark for cybersecurity, assessing knowledge that could aid in cyber weapons development.","url":"https://www.wmdp.ai/","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/centerforaisafety/wmdp","paper":"https://arxiv.org/abs/2403.03218","modality":"text","metric":"percent","higher_is_better":false,"classification":["Knowledge","QA","Text"]},"wmt16":{"id":"wmt16","name":"WMT16 En–De","topic":"Machine translation","description":"WMT16 English–German translation benchmark (news).","url":"https://www.statmt.org/wmt16/translation-task.html","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"bleu","higher_is_better":true,"classification":["Translation","Text"]},"wmt16-instruct":{"id":"wmt16-instruct","name":"WMT16 En–De (Instruct)","topic":"Machine translation","description":"Instruction-tuned evaluation on the WMT16 English–German translation set.","url":"","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"bleu","higher_is_better":true,"classification":["Translation","Text"]},"wmt20-en-ja":{"id":"wmt20-en-ja","name":"WMT20 En-Ja","topic":"Machine translation","description":"WMT20 English-to-Japanese news translation (Swallow eval, BLEU).","url":"https://swallow-llm.github.io/leaderboard/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"bleu","higher_is_better":true,"classification":["Translation","Japanese","Text"]},"wmt20-ja-en":{"id":"wmt20-ja-en","name":"WMT20 Ja-En","topic":"Machine translation","description":"WMT20 Japanese-to-English news translation (Swallow eval, BLEU).","url":"https://swallow-llm.github.io/leaderboard/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"bleu","higher_is_better":true,"classification":["Translation","Japanese","Text"]},"wmt23-ende":{"id":"wmt23-ende","name":"WMT23 En–De","topic":"Machine translation","description":"WMT23 English–German translation benchmark.","url":"https://machinetranslate.org/wmt23","stars":"","updated_at":"2025-09-08","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"wmt24-plus-plus":{"id":"wmt24-plus-plus","name":"WMT24++","topic":"Machine translation","description":"Extended WMT 2024 evaluation across multiple language pairs.","url":"https://arxiv.org/abs/2502.12404v1","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"wod-e2e":{"id":"wod-e2e","name":"WOD-E2E (RFS)","topic":"Driving planning","description":"Waymo Open Dataset end-to-end driving challenge on long-tail scenarios; score = Rater Feedback Score (RFS, 0-10) on the test split.","url":"https://waymo.com/open/challenges/2025/e2e-driving/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"score","higher_is_better":true,"classification":["Autonomous Driving","Vision"]},"workbench":{"id":"workbench","name":"WorkBench","topic":"Agentic tool use","description":"Workplace agent benchmark: tasks over simulated email, calendar, CRM, analytics and project-management tools; scored by task completion (outcome-based).","url":"https://arxiv.org/abs/2405.00823","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tool Use","Text"]},"workbuddy-bench":{"id":"workbuddy-bench","name":"WorkBuddy Bench","topic":"Office assistant agents","description":"Agentic benchmark of workplace-assistant (WorkBuddy) tasks such as documents, spreadsheets and messaging.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"workspacebench":{"id":"workspacebench","name":"Workspace Bench","topic":"Agentic office work","description":"Workspace Bench: OpenClaw-style workspace tasks with large heterogeneous file dependencies (100-task setting).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"workspacebench-lite":{"id":"workspacebench-lite","name":"Workspace Bench Lite","topic":"Agentic office work","description":"Lite subset of Workspace Bench: workspace tasks with heterogeneous file dependencies.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Text"]},"worldbench":{"id":"worldbench","name":"WorldBench","topic":"Visual knowledge","description":"Benchmark of real-world visual knowledge and recognition.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Vision","Knowledge"]},"worldtravel2-multimodal":{"id":"worldtravel2-multimodal","name":"WorldTravel2 (multi-modal)","topic":"Travel planning (multimodal)","description":"WorldTravel2 benchmark multimodal track.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Travel","Multimodal","Vision","Text"]},"worldtravel2-text":{"id":"worldtravel2-text","name":"WorldTravel2 (text)","topic":"Travel planning (text)","description":"WorldTravel2 benchmark text-only track.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Travel","Text"]},"worldvqa":{"id":"worldvqa","name":"WorldVQA","topic":"World knowledge VQA","description":"Visual question answering requiring world knowledge and commonsense reasoning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Knowledge","Multimodal","QA","Vision"]},"worldvqa-with-tools":{"id":"worldvqa-with-tools","name":"WorldVQA (With Tools)","topic":"Visual world knowledge","description":"WorldVQA evaluated with tool access in an agent harness.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":"Multimodal"},"writingbench":{"id":"writingbench","name":"WritingBench","topic":"Writing quality","description":"General-purpose writing quality benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"wsc":{"id":"wsc","name":"WSC","topic":"Coreference reasoning","description":"Classic Winograd Schema Challenge measuring commonsense coreference.","url":"https://cs.nyu.edu/faculty/davise/papers/WinogradSchemas/WS.html","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"https://arxiv.org/abs/1109.1334","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"wtq":{"id":"wtq","name":"WikiTableQuestions","topic":"Table QA","description":"WikiTableQuestions (MMTab subset, image tables).","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["QA","Vision"]},"xbench-deepsearch":{"id":"xbench-deepsearch","name":"xBench-DeepSearch","topic":"Agentic research","description":"Evaluates multi-hop deep research workflows on xBench DeepSearch tasks.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"avg@3","higher_is_better":true,"classification":["Agents","Research","Text"]},"xbench-deepsearch-2025.05":{"id":"xbench-deepsearch-2025.05","name":"xBench-DeepSearch (2025.05)","topic":"Agentic research","description":"xBench DeepSearch benchmark May 2025 snapshot.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"avg@3","higher_is_better":true,"classification":["Agents","Research","Text"]},"xbench-deepsearch-2025.10":{"id":"xbench-deepsearch-2025.10","name":"xBench-DeepSearch (2025.10)","topic":"Agentic research","description":"xBench DeepSearch benchmark October 2025 snapshot.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"avg@3","higher_is_better":true,"classification":["Agents","Research","Text"]},"xcopa":{"id":"xcopa","name":"XCOPA","topic":"Cross-lingual commonsense causal reasoning","description":"Multilingual extension of COPA causal commonsense reasoning in 11 languages.","url":"https://huggingface.co/datasets/cambridgeltl/xcopa","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Reasoning","Text"]},"xdev-bench-hard":{"id":"xdev-bench-hard","name":"xdev-bench-hard","topic":"Cybersecurity","description":"Moonshot internal offensive-security / exploit-development benchmark; problems solved (pass@3).","url":"https://www.kimi.ai/blog/kimi-k3-1-pro","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"pass@3","higher_is_better":true,"classification":["Code","Agents","Safety","Text"]},"xlrs-bench":{"id":"xlrs-bench","name":"XLRS-Bench","topic":"Remote sensing","description":"Remote sensing benchmark for evaluating multimodal AI on satellite and aerial imagery.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Remote Sensing","Science","Vision"]},"xnli":{"id":"xnli","name":"XNLI","topic":"Cross-lingual NLI","description":"Cross-lingual natural language inference benchmark (MultiNLI translated into 15 languages).","url":"https://huggingface.co/datasets/facebook/xnli","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Reasoning","Text"]},"xorqa-in":{"id":"xorqa-in","name":"XorQA-IN","topic":"Indic languages","description":"Cross-lingual open-retrieval QA in Indian languages (IndicXTREME), F1.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"xpertbench-edu":{"id":"xpertbench-edu","name":"XpertBench (Edu)","topic":"Economics/education","description":"XpertBench education domain subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Education","Text"]},"xpertbench-fin":{"id":"xpertbench-fin","name":"XpertBench (Fin)","topic":"Economics/finance","description":"XpertBench finance domain subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Finance","Text"]},"xpertbench-humanities":{"id":"xpertbench-humanities","name":"XpertBench (Humanities)","topic":"Economics/humanities","description":"XpertBench humanities domain subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Text"]},"xpertbench-law":{"id":"xpertbench-law","name":"XpertBench (Law)","topic":"Economics/legal","description":"XpertBench legal domain subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Law","Text"]},"xpertbench-research":{"id":"xpertbench-research","name":"XpertBench (Research)","topic":"Economics/research","description":"XpertBench research domain subset.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Economics","Research","Text"]},"xquad":{"id":"xquad","name":"XQuAD","topic":"Multilingual QA","description":"Cross-lingual Question Answering Dataset: SQuAD v1.1 subset professionally translated into 10 languages.","url":"https://github.com/google-deepmind/xquad","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","QA","Text"]},"xquad-in":{"id":"xquad-in","name":"XQuAD-IN","topic":"Indic languages","description":"Extractive QA (XQuAD) in Indian languages (IndicXTREME), F1.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Multilingual","Text"]},"xstest":{"id":"xstest","name":"XSTest","topic":"Safety","description":"XSTest safety benchmark.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Safety","Text"]},"xstorycloze":{"id":"xstorycloze","name":"XStoryCloze","topic":"Commonsense reasoning (multilingual)","description":"Multilingual StoryCloze: choose the correct story ending in 11 languages.","url":"https://huggingface.co/datasets/juletxara/xstory_cloze","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2112.10668","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Multilingual","Text"]},"xwinograd":{"id":"xwinograd","name":"XWinograd","topic":"Coreference reasoning (multilingual)","description":"Multilingual Winograd schema benchmark (en, fr, jp, pt, ru, zh).","url":"https://huggingface.co/datasets/Muennighoff/xwinograd","stars":"","updated_at":"","leaderboard":"","github":"","paper":"https://arxiv.org/abs/2211.01786","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Multilingual","Text"]},"yc-bench":{"id":"yc-bench","name":"YC-Bench","topic":"Long-horizon business simulation","description":"YC-Bench: long-horizon startup-management simulation; metric is final assets of the fund in millions of USD.","url":"","stars":"","updated_at":"","leaderboard":"","github":"https://github.com/collinear-ai/yc-bench","paper":"","modality":"text","metric":"usd_millions","higher_is_better":true,"classification":["Agents","Text"]},"zclawbench":{"id":"zclawbench","name":"ZClawBench","topic":"Agent tasks (OpenClaw)","description":"Z.ai end-to-end benchmark for OpenClaw agent workflows (environment setup, software dev, retrieval, data analysis, content creation).","url":"https://docs.z.ai/guides/llm/glm-5-turbo","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Agents","Tools","Text"]},"zebralogic":{"id":"zebralogic","name":"ZebraLogic","topic":"Logical reasoning","description":"Logical reasoning benchmark assessing complex pattern and rule inference.","url":"","stars":"","updated_at":"","leaderboard":"https://huggingface.co/spaces/allenai/ZeroEval","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Reasoning","Text"]},"zerobench":{"id":"zerobench","name":"ZeroBench","topic":"Zero-shot generalization","description":"Evaluates zero-shot performance across diverse tasks without task-specific finetuning.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"zerobench-sub":{"id":"zerobench-sub","name":"ZeroBench (sub)","topic":"Zero-shot generalization","description":"Subset of ZeroBench targeting harder zero-shot reasoning cases.","url":"","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"percent","higher_is_better":true,"classification":["Text"]},"zerobench-with-tools":{"id":"zerobench-with-tools","name":"ZeroBench (With Tools)","topic":"Visual reasoning","description":"ZeroBench main set evaluated with tool access in an agent harness (pass@5).","url":"https://zerobench.github.io/","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"visual","metric":"percent","higher_is_better":true,"classification":["Reasoning","Vision"]},"zeroscrolls-booksumsort":{"id":"zeroscrolls-booksumsort","name":"ZeroSCROLLS BookSumSort","topic":"Long-context summarization","description":"ZeroSCROLLS split based on BookSumSort long-form summarization.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"cidx","higher_is_better":true,"classification":["Long Context","Summarization","Text"]},"zeroscrolls-govreport":{"id":"zeroscrolls-govreport","name":"ZeroSCROLLS GovReport","topic":"Long-context summarization","description":"ZeroSCROLLS split based on the GovReport summarization benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"rgeo","higher_is_better":true,"classification":["Long Context","Summarization","Text"]},"zeroscrolls-musique":{"id":"zeroscrolls-musique","name":"ZeroSCROLLS MuSiQue","topic":"Long-context reasoning","description":"ZeroSCROLLS split derived from MuSiQue multi-hop QA.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Long Context","Reasoning","Text"]},"zeroscrolls-narrativeqa":{"id":"zeroscrolls-narrativeqa","name":"ZeroSCROLLS NarrativeQA","topic":"Long-context QA","description":"ZeroSCROLLS split based on the NarrativeQA reading comprehension benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Long Context","QA","Text"]},"zeroscrolls-qasper":{"id":"zeroscrolls-qasper","name":"ZeroSCROLLS Qasper","topic":"Long-context QA","description":"ZeroSCROLLS split based on the Qasper paper QA benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"f1","higher_is_better":true,"classification":["Long Context","QA","Text"]},"zeroscrolls-qmsum":{"id":"zeroscrolls-qmsum","name":"ZeroSCROLLS QMSum","topic":"Long-context summarization","description":"ZeroSCROLLS split based on the QMSum meeting summarization benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"rgeo","higher_is_better":true,"classification":["Long Context","Summarization","Text"]},"zeroscrolls-quality":{"id":"zeroscrolls-quality","name":"ZeroSCROLLS QuALITY","topic":"Long-context QA","description":"ZeroSCROLLS split based on the QuALITY reading comprehension benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"ac","higher_is_better":true,"classification":["Long Context","QA","Text"]},"zeroscrolls-spacedigest":{"id":"zeroscrolls-spacedigest","name":"ZeroSCROLLS SpaceDigest","topic":"Long-context summarization","description":"ZeroSCROLLS SpaceDigest extractive summarization task.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"es","higher_is_better":true,"classification":["Long Context","Summarization","Text"]},"zeroscrolls-squality":{"id":"zeroscrolls-squality","name":"ZeroSCROLLS SQuALITY","topic":"Long-context summarization","description":"ZeroSCROLLS split based on the SQuALITY long-form summarization benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"2025-10-28","leaderboard":"","github":"","paper":"","modality":"text","metric":"rouge_gm","higher_is_better":true,"classification":["Long Context","Summarization","Text"]},"zeroscrolls-summscreenfd":{"id":"zeroscrolls-summscreenfd","name":"ZeroSCROLLS SummScreenFD","topic":"Long-context summarization","description":"ZeroSCROLLS split based on the SummScreenFD summarization benchmark.","url":"https://github.com/allenai/scrolls","stars":"","updated_at":"","leaderboard":"","github":"","paper":"","modality":"text","metric":"rgeo","higher_is_better":true,"classification":["Long Context","Summarization","Text"]}}