from dataclasses import dataclass, field
from typing import Literal

from datasets import load_dataset


@dataclass
class BenchQuestion:
    bench_name: str
    qid: str
    question: str
    choices: list[str] | None  # List if multiple choice, None if free response
    gold_answer: str  # "A"/"B"/... for multiple choice, rubric JSON for free response
    metadata: dict = field(default_factory=dict)


def load_medqa(split: str = "test", max_samples: int | None = None) -> list[BenchQuestion]:
    """Load MedQA (USMLE style). Repository: bigbio/med_qa."""
    ds = load_dataset("bigbio/med_qa", "med_qa_en_source", split=split)
    if max_samples:
        ds = ds.select(range(min(max_samples, len(ds))))
    questions = []
    for i, row in enumerate(ds):
        questions.append(BenchQuestion(
            bench_name="medqa",
            qid=f"medqa_{i}",
            question=row["question"],
            choices=[opt["value"] for opt in row["options"]],
            gold_answer=row["answer_idx"],  # "A", "B", ...
            metadata={"meta_info": row.get("meta_info", "")},
        ))
    return questions


def load_pubmedqa(split: str = "train", max_samples: int | None = None) -> list[BenchQuestion]:
    """Load PubMedQA (yes/no/maybe). Repository: qiaojin/PubMedQA labeled subset."""
    ds = load_dataset("qiaojin/PubMedQA", "pqa_labeled", split=split)
    if max_samples:
        ds = ds.select(range(min(max_samples, len(ds))))
    questions = []
    for i, row in enumerate(ds):
        context = " ".join(row["context"]["contexts"])
        questions.append(BenchQuestion(
            bench_name="pubmedqa",
            qid=f"pmqa_{i}",
            question=f"Context: {context}\n\nQuestion: {row['question']}\nAnswer one of: yes/no/maybe:",
            choices=["yes", "no", "maybe"],
            gold_answer=row["final_decision"],
            metadata={"pmid": row.get("pubid", "")},
        ))
    return questions


def load_mmlu_medical(split: str = "test", max_samples: int | None = None) -> list[BenchQuestion]:
    """Load MMLU's medical subset. Repository: cais/mmlu."""
    subsets = ["clinical_knowledge", "college_medicine", "anatomy", "professional_medicine", "medical_genetics"]
    questions = []
    for subset in subsets:
        ds = load_dataset("cais/mmlu", subset, split=split)
        if max_samples:
            ds = ds.select(range(min(max_samples // len(subsets), len(ds))))
        for i, row in enumerate(ds):
            questions.append(BenchQuestion(
                bench_name=f"mmlu_{subset}",
                qid=f"mmlu_{subset}_{i}",
                question=row["question"],
                choices=row["choices"],
                gold_answer=["A", "B", "C", "D"][row["answer"]],
                metadata={"subset": subset},
            ))
    return questions


def load_medmcqa(split: str = "validation", max_samples: int | None = None) -> list[BenchQuestion]:
    """Load MedMCQA (Indian medical exam). Repository: openlifescienceai/medmcqa."""
    try:
        ds = load_dataset("openlifescienceai/medmcqa", split=split)
    except Exception:
        return []
    if max_samples:
        ds = ds.select(range(min(max_samples, len(ds))))
    questions = []
    for i, row in enumerate(ds):
        questions.append(BenchQuestion(
            bench_name="medmcqa",
            qid=f"medmcqa_{i}",
            question=row["question"],
            choices=[row["opa"], row["opb"], row["opc"], row["opd"]],
            gold_answer=["A", "B", "C", "D"][row["cop"]],
            metadata={"subject_name": row.get("subject_name", "")},
        ))
    return questions


def load_healthbench(split: str = "test", max_samples: int | None = None) -> list[BenchQuestion]:
    """Load HealthBench (OpenAI 2025). Repository and license need to be checked in advance.
    
    The actual repository name needs to be confirmed at the time of publication. This is a conceptual stub.
    """
    try:
        ds = load_dataset("openai/healthbench", split=split)
    except Exception:
        # Alternative: Load sample scenarios from the paper's appendix manually
        return []
    if max_samples:
        ds = ds.select(range(min(max_samples, len(ds))))
    questions = []
    for i, row in enumerate(ds):
        questions.append(BenchQuestion(
            bench_name="healthbench",
            qid=f"hb_{i}",
            question=row["scenario"],
            choices=None,
            gold_answer=row.get("rubric_json", "{}"),
            metadata=row.get("metadata", {}),
        ))
    return questions

ZERO_SHOT_MEDQA = """Answer the following medical national exam question.

Question: {question}

Choices:
{choices_formatted}

Answer (only one letter: A/B/C/D/E):
"""

ZERO_SHOT_COT_MEDQA = """Answer the following medical national exam question. First, reason step-by-step, and then provide the final answer.

Question: {question}

Choices:
{choices_formatted}

Step-by-step reasoning:
"""

FEW_SHOT_MEDQA = """Here are some examples of medical national exam questions and their answers.

Example 1:
Question: A 65-year-old male presents with sudden onset of chest pain, diaphoresis, and nausea for 3 hours. EKG shows ST elevation. What is the most likely diagnosis?
Choices: A. Acute myocardial infarction B. Unstable angina C. Aortic dissection D. Pulmonary embolism E. Pericarditis
Answer: A

Example 2:
Question: A 45-year-old female presents with night sweats, weight loss, and chronic cough for 3 months. Chest X-ray shows an upper lobe nodule. What is the most appropriate initial test?
Choices: A. CT scan B. Sputum acid-fast bacilli smear C. Tuberculin skin test D. Bronchoscopy E. Mantoux test
Answer: B

Now answer the following question.

Question: {question}

Choices:
{choices_formatted}

Answer (one letter only):
"""

ZERO_SHOT_PUBMEDQA = """{question}
"""

ZERO_SHOT_MMLU = """Answer the following question.

Question: {question}

Choices:
A. {a}
B. {b}
C. {c}
D. {d}

Answer (one letter only: A/B/C/D):
"""

def format_question(q: BenchQuestion, prompt_type: str = "zero_shot") -> str:
    """Render prompts for each benchmark."""
    if q.bench_name == "medqa":
        choices_text = "\n".join(f"{chr(65+i)}. {c}" for i, c in enumerate(q.choices or []))
        template = {
            "zero_shot": ZERO_SHOT_MEDQA,
            "zero_shot_cot": ZERO_SHOT_COT_MEDQA,
            "few_shot": FEW_SHOT_MEDQA,
        }.get(prompt_type, ZERO_SHOT_MEDQA)
        return template.format(question=q.question, choices_formatted=choices_text)
    
    elif q.bench_name == "pubmedqa":
        return ZERO_SHOT_PUBMEDQA.format(question=q.question)
    
    elif q.bench_name.startswith("mmlu_"):
        return ZERO_SHOT_MMLU.format(
            question=q.question,
            a=q.choices[0], b=q.choices[1], c=q.choices[2], d=q.choices[3],
        )
    
    elif q.bench_name == "medmcqa":
        choices_text = "\n".join(f"{chr(65+i)}. {c}" for i, c in enumerate(q.choices or []))
        return ZERO_SHOT_MEDQA.format(question=q.question, choices_formatted=choices_text)
    
    elif q.bench_name == "healthbench":
        return q.question  # Free response
    
    else:
        raise ValueError(f"Unknown bench: {q.bench_name}")

import re


def extract_choice(response: str, choices: list[str] | None) -> str | None:
    """Extract one of the choices (A/B/C/D) from the LLM response.

    Handles CoT responses (long reasoning followed by an answer).
    """
    if not choices:
        return None
    upper = response.upper()
    # First, try patterns like "Answer: A", "The answer is A", etc.
    for pattern in [
        r"Answer[:\s]+([A-E])",
        r"정답[은:\s]+([A-E])",
        r"answer[:\s]+([A-E])",
        r"the answer is\s+([A-E])",
        r"final answer[:\s]+([A-E])",
    ]:
        m = re.search(pattern, upper)
        if m:
            return m.group(1)
    # Fallback: Last standalone A~E
    matches = re.findall(r"\b([A-E])\b", upper)
    if matches:
        return matches[-1]
    # Fallback: Match choice text
    for i, choice in enumerate(choices):
        if choice and choice.lower() in response.lower():
            return chr(65 + i)
    return None


def extract_yes_no_maybe(response: str) -> str | None:
    """Extract yes/no/maybe for PubMedQA."""
    text = response.lower().strip()
    for keyword in ["yes", "no", "maybe"]:
        if re.search(rf"\b{keyword}\b", text[:100]):
            return keyword
    return None

JUDGE_PROMPT = """You are an expert in clinical diagnosis and treatment evaluation.
Evaluate the response based on the rubric for the given scenario.

Scenario:
{scenario}

Response:
{response}

Rubric (JSON, each item is either 0 or 1):
{rubric}

Output a JSON with the score (0 or 1) for each rubric item:
{{
  "criterion_1": 0 or 1,
  "criterion_2": 0 or 1,
  ...
  "total_score": total,
  "max_score": total number of rubric items
}}
"""

def judge_healthbench_response(
    scenario: str,
    response: str,
    rubric_json: str,
    judge_client,
    judge_model: str = "claude-opus-4-5-20250219",
) -> dict:
    """Score rubric using LLM-as-judge."""
    prompt = JUDGE_PROMPT.format(scenario=scenario, response=response, rubric=rubric_json)
    resp = judge_client.messages.create(
        model=judge_model,
        max_tokens=1024,
        messages=[{"role": "user", "content": prompt}],
    )
    raw = resp.content[0].text
    match = re.search(r"\{.*\}", raw, re.DOTALL)
    if not match:
        return {"total_score": 0, "max_score": 1, "error": "Failed to parse judgment"}
    import json
    try:
        return json.loads(match.group(0))
    except json.JSONDecodeError:
        return {"total_score": 0, "max_score": 1, "error": "Invalid JSON"}

import asyncio
from dataclasses import dataclass


@dataclass
class BenchAnswerResult:
    question: BenchQuestion
    model: str
    model_snapshot: str
    response: str
    predicted_answer: str | None
    is_correct: bool
    latency_ms: float
    metadata: dict


async def run_bench_on_vendor(
    questions: list[BenchQuestion],
    adapter,  # VendorAdapter from Part 09
    prompt_type: str = "zero_shot",
    max_concurrent: int = 5,
) -> list[BenchAnswerResult]:
    """Run a single benchmark on a single model."""
    semaphore = asyncio.Semaphore(max_concurrent)
    
    async def process(q: BenchQuestion) -> BenchAnswerResult:
        async with semaphore:
            prompt = format_question(q, prompt_type=prompt_type)
            resp = await adapter.query_async(prompt)
            if q.choices:
                if q.bench_name == "pubmedqa":
                    predicted = extract_yes_no_maybe(resp.raw_text)
                else:
                    predicted = extract_choice(resp.raw_text, q.choices)
                is_correct = predicted == q.gold_answer
            else:
                # For free-response tasks like HealthBench, use a separate judge
                predicted = None
                is_correct = False
            return BenchAnswerResult(
                question=q, model=adapter.model, model_snapshot=resp.model_snapshot,
                response=resp.raw_text, predicted_answer=predicted,
                is_correct=is_correct, latency_ms=resp.latency_ms,
                metadata={
                    "input_tokens": resp.input_tokens,
                    "output_tokens": resp.output_tokens,
                    "prompt_type": prompt_type,
                },
            )
    
    return await asyncio.gather(*[process(q) for q in questions])

import numpy as np
import pandas as pd


# Vendor numbers and published numbers (need to be confirmed at runtime)
PUBLISHED_NUMBERS = {
    "medqa": {
        "claude-opus-4-5": 0.905,  # Example, published by Anthropic
        "gpt-4o": 0.876,  # Published by OpenAI
        "o1": 0.947,  # Published by OpenAI o1
        "gemini-2.5-pro": 0.895,  # Published by Google
        "med-gemini": 0.91,  # Google Med-Gemini
        "meditron-70b": 0.72,  # Published in EPFL paper
        "med42-70b": 0.85,  # Published by M42
    },
    "pubmedqa": {
        "claude-opus-4-5": 0.788,
        "gpt-4o": 0.759,
        "o1": 0.82,
        "meditron-70b": 0.82,  # Specialized models perform better
        "med-gemini": 0.81,
    },
    "mmlu_medical": {
        "gpt-4o": 0.87,
        "claude-opus-4-5": 0.89,
        "gemini-2.5-pro": 0.88,
        "meditron-70b": 0.75,
    },
    # ... other benchmarks
}


def compute_bench_score(results: list[BenchAnswerResult]) -> dict:
    """Calculate accuracy and 95% CI (Wilson score interval)."""
    n = len(results)
    if n == 0:
        return {"accuracy": 0.0, "ci_low": 0.0, "ci_high": 0.0, "n": 0}
    correct = sum(1 for r in results if r.is_correct)
    p = correct / n
    z = 1.96
    denominator = 1 + z**2 / n
    center = (p + z**2 / (2 * n)) / denominator
    margin = z * np.sqrt(p * (1 - p) / n + z**2 / (4 * n**2)) / denominator
    return {
        "accuracy": p,
        "ci_low": max(0, center - margin),
        "ci_high": min(1, center + margin),
        "n": n,
        "correct": correct,
    }


def delta_report(all_results: dict[tuple[str, str], list[BenchAnswerResult]]) -> pd.DataFrame:
    """(bench, model) -> delta between measured and published."""
    rows = []
    for (bench, model), results in all_results.items():
        score = compute_bench_score(results)
        # Match model alias from published numbers (needs to be confirmed at runtime)
        model_key = model.split("-2024")[0].split("-2025")[0]  # Remove snapshot
        published = PUBLISHED_NUMBERS.get(bench, {}).get(model_key)
        delta = (score["accuracy"] - published) if published is not None else None
        
        reproducibility = "N/A"
        if delta is not None:
            if abs(delta) < 0.02:
                reproducibility = "Reproduced (Δ < 2%p)"
            elif abs(delta) < 0.05:
                reproducibility = "Partially reproduced (Δ 2~5%p)"
            else:
                reproducibility = f"Failed to reproduce (Δ {delta:.1%})"
        
        rows.append({
            "bench": bench,
            "model": model,
            "n": score["n"],
            "correct": score["correct"],
            "accuracy_measured": round(score["accuracy"], 4),
            "ci_95": f"[{score['ci_low']:.3f}, {score['ci_high']:.3f}]",
            "published": published,
            "delta": round(delta, 4) if delta is not None else None,
            "reproducibility": reproducibility,
        })
    return pd.DataFrame(rows)


def reproducibility_heatmap(df: pd.DataFrame, output_path: str = "reproducibility.png") -> None:
    """Benchmark x model heatmap (delta color)."""
    import matplotlib.pyplot as plt
    import seaborn as sns
    pivot = df.pivot(index="model", columns="bench", values="delta")
    fig, ax = plt.subplots(figsize=(12, 8))
    sns.heatmap(
        pivot, annot=True, cmap="RdYlGn_r", center=0, fmt=".3f",
        cbar_kws={"label": "Delta (Measured - Published)"},
        ax=ax,
    )
    ax.set_title("Med-LLM Benchmark Reproduction Delta Matrix\n(Positive = Measured better, Negative = Failed to reproduce)")
    plt.tight_layout()
    plt.savefig(output_path, dpi=150)

def diagnose_reproduction_failure(
    bench: str,
    model: str,
    delta: float,
    all_results: list[BenchAnswerResult],
) -> dict:
    """Diagnose possible causes of reproduction failure."""
    diagnosis = {
        "bench": bench, "model": model, "delta": delta,
        "possible_causes": [],
    }
    
    if abs(delta) < 0.03:
        diagnosis["possible_causes"].append("Reproduced. No further investigation needed.")
        return diagnosis
    
    # 1. Prompt differences
    prompt_types = {r.metadata.get("prompt_type", "unknown") for r in all_results}
    if "zero_shot_cot" not in prompt_types:
        diagnosis["possible_causes"].append(
            "CoT (Chain-of-Thought) prompt not used. The original paper likely used CoT. Re-run with the zero_shot_cot prompt."
        )
    if "few_shot" not in prompt_types:
        diagnosis["possible_causes"].append(
            "Few-shot examples not included. The original paper likely used 5-shot, 25-shot, etc."
        )
    
    # 2. Data subset differences
    total_n = len(all_results)
    diagnosis["possible_causes"].append(
        f"Current sample size: n={total_n}. The original paper likely used the full test set (thousands of samples). Increase sample size."
    )
    
    # 3. Self-consistency not used
    diagnosis["possible_causes"].append(
        "Self-consistency (majority vote of N samples) not used. The original paper likely used N=5, 10, etc."
    )
    
    # 4. Data contamination
    diagnosis["possible_causes"].append(
        f"{bench} is an open dataset. It is possible that the model's pre-training corpus included this data (data contamination). Recommend validating with a more recent, held-out benchmark."
    )
    
    # 5. Model snapshot differences
    snapshots = {r.model_snapshot for r in all_results}
    diagnosis["possible_causes"].append(
        f"Measured snapshots: {snapshots}. May differ from the snapshot used at the time of publication."
    )
    
    return diagnosis

async def full_reproduction_bench(
    benches: list[str],  # ["medqa", "pubmedqa", "mmlu", "medmcqa"]
    adapters: list,  # List of VendorAdapter
    output_dir: Path,
    sample_size: int | None = 500,
    prompt_types: list[str] = ["zero_shot"],
) -> pd.DataFrame:
    """Run the full reproduction benchmark."""
    output_dir.mkdir(parents=True, exist_ok=True)
    
    # Load benchmarks
    all_questions = {}
    if "medqa" in benches:
        all_questions["medqa"] = load_medqa(max_samples=sample_size)
    if "pubmedqa" in benches:
        all_questions["pubmedqa"] = load_pubmedqa(max_samples=sample_size)
    if "mmlu" in benches:
        all_questions["mmlu_medical"] = load_mmlu_medical(max_samples=sample_size)
    if "medmcqa" in benches:
        all_questions["medmcqa"] = load_medmcqa(max_samples=sample_size)
    
    print(f"Finished loading. Samples per benchmark: " +
          ", ".join(f"{k}={len(v)}" for k, v in all_questions.items()))
    
    all_results = {}
    for adapter in adapters:
        for bench_name, questions in all_questions.items():
            for pt in prompt_types:
                print(f"[{adapter.model}] {bench_name} ({pt})")
                results = await run_bench_on_vendor(questions, adapter, prompt_type=pt)
                key = (bench_name, adapter.model)
                if pt != "zero_shot":
                    key = (f"{bench_name}_{pt}", adapter.model)
                all_results[key] = results
    
    # Report
    df = delta_report(all_results)
    df.to_csv(output_dir / "reproduction_report.csv", index=False)
    reproducibility_heatmap(df, str(output_dir / "reproducibility_heatmap.png"))
    
    # Failure diagnosis
    diagnoses = []
    for (bench, model), results in all_results.items():
        row = df[(df["bench"] == bench) & (df["model"] == model)]
        if not row.empty and row.iloc[0]["delta"] is not None:
            diag = diagnose_reproduction_failure(bench, model, row.iloc[0]["delta"], results)
            diagnoses.append(diag)
    
    with open(output_dir / "failure_diagnoses.json", "w", encoding="utf-8") as f:
        import json
        json.dump(diagnoses, f, ensure_ascii=False, indent=2)
    
    return df

## Performance, Cost, and Known Failure Cases

### Performance Reference (Using Public Benchmarks)

Med-LLM benchmark recent results (approximate, model-by-year):

| Model | MedQA (USMLE) | PubMedQA | MMLU-Medical | MedMCQA | HealthBench | Source |
|------|:-------------:|:--------:|:------------:|:-------:|:-----------:|------|
| Meditron 70B | 0.72 | 0.82 | 0.75 | 0.65 | — | Chen et al., EPFL 2023 [6] |
| PMC-LLaMA 13B | 0.61 | 0.77 | 0.65 | 0.60 | — | Wu et al. 2023 [7] |
| Med-PaLM 2 | 0.86 | 0.79 | 0.85 | 0.72 | — | Singhal et al., Nature 2023 [8] |
| Med-Gemini | 0.91 | 0.81 | 0.88 | 0.75 | 0.62 | Saab et al., Nat Med 2024 [9] |
| Med42 70B | 0.85 | 0.79 | 0.83 | 0.71 | — | M42 release |
| GPT-4o | 0.88 | 0.76 | 0.87 | 0.72 | 0.65 | OpenAI release [10] |
| Claude Opus 4.5 | 0.91 | 0.79 | 0.89 | 0.76 | 0.72 | Anthropic release [11] |
| o1 | 0.95+ | 0.82 | 0.92 | 0.83 | 0.78 | OpenAI release [10] |
| o3 | 0.94 | 0.82 | 0.91 | 0.81 | 0.79 | OpenAI release [10] |

### Estimated Cost for Learners to Reproduce

- 6-8 models × 4 benchmarks × 500 samples = 12,000-16,000 requests, approximately 30-120 USD total (check the pricing for each model at the time).
- Separate GPU time is required when running open models locally.

### 5 Known Failure Cases (Collected from the Community and Papers)

1. **Benchmark Data Contamination**
   Symptoms: The model may have seen the benchmark questions during the pre-training phase → inflated accuracy in the published results. If the reproduced results are unexpectedly lower, it is not necessarily a reproduction failure (due to differences in prompts or subsets).
   Cause: MedQA and PubMedQA are old, open datasets, so they likely include web-crawled corpora.
   Mitigation: (a) Prioritize newer benchmarks (HealthBench, etc.), (b) use contamination detection tools (n-gram overlap, membership inference), (c) create a held-out benchmark with new questions, (d) use a pre-training snapshot from before the benchmark release date.
   Source: Xu et al. "Benchmark Data Contamination of Large Language Models." arXiv 2024 [12].

2. **Accuracy Varies by More Than 5% with Slight Differences in Prompts**
   Symptoms: Accuracy drops or increases significantly even with slight changes to the original paper's prompt.
   Cause: LLMs are sensitive to prompt formatting. The order, number, and domain relevance of the few-shot examples all have an impact.
   Mitigation: (a) Copy the original paper's prompt exactly from the appendix and rerun the code from the original repository, (b) ensemble multiple prompt variations (self-consistency N=5, 10), (c) save the results along with the prompt and parameter logs, (d) pin the benchmark execution conditions (temperature=0, etc.).
   Source: Lu et al. "Fantastically Ordered Prompts and Where to Find Them." ACL 2022 [13].

3. **LLM-as-Judge Bias (HealthBench, etc.)**
   Symptoms: The judge model scores responses from a specific vendor's model more favorably (self-preference bias).
   Cause: If the judge and the model being scored are from the same lineage, the judge may prefer its own style.
   Mitigation: (a) Cross-validate with judges and target models from different vendors, (b) ensemble multiple judges (Claude + GPT + Gemini), (c) periodically check the correlation with human scoring, (d) use rubric-based scoring (minimize subjective judgment).
   Source: Zheng et al. "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena." NeurIPS 2023 [14].

4. **Significant Accuracy Difference Depending on Whether CoT is Used**
   Symptoms: The measured accuracy of GPT-4o on MedQA in a zero-shot setting is 0.79, but with a CoT prompt, it is 0.88 (a 9% difference).
   Cause: Many medical QA problems require multi-step reasoning. Without CoT, the model may rely on surface-level pattern matching and provide incorrect answers.
   Mitigation: (a) Report all optimal prompts for each benchmark (zero-shot / CoT / few-shot / self-consistency), (b) confirm the explicit prompt conditions in the published paper, (c) models like o1 and o3, which have their own CoT, may have zero-shot as the optimal setting.
   Source: Wei et al. "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models." NeurIPS 2022 [15].

5. **Dataset License and Access Issues**
   Symptoms: Certain benchmarks, such as HealthBench and MedMCQA, require registration and authentication, making it impossible for learners to access them immediately.
   Cause: Clinical data and medical exam questions have copyright and regulatory issues.
   Mitigation: (a) Check the license for each benchmark in advance and specify the registration procedure, (b) prioritize benchmarks with open licenses (MMLU, PubMedQA, etc.), (c) supplement with a self-created held-out benchmark, (d) guide learners through the registration process.
   Source: HuggingFace Hub data card for each dataset, license section of the original paper.

## Expansion Ideas

- **Korean Medical Benchmark:** Build and reproduce a KMLE (Korean Medical Licensing Exam-style) benchmark. Validate it specifically for Korea.
- **Temporal Robustness:** Rerun the same benchmark monthly to track updates to vendor models. Detect regressions.
- **Task-Specific Fine-tuning Benchmark:** Quantify the performance improvement of open models after fine-tuning.
- **Chain-of-Thought Comparison:** Matrix showing the impact of CoT prompt, few-shot N, and self-consistency N on benchmark performance.
- **RAG Integration:** Search for relevant PubMed abstracts for benchmark questions and inject them as context → quantify the performance improvement.
- **Large-Scale Benchmark of Open Models:** Encompass all open-source models, including Meditron, Med42, MedGemma, Llama-3-Med, and Qwen-Med.

## Next Section

- Section 09 `llm-vendor-benchmark`: If this section is about reproducing standard benchmarks, section 09 will be about comparing vendors in real-world tasks.
- Section 14 `bio-mcp-agent`: Use the results of this benchmark to expose them in the MCP tool.
- Section 01 `clinical-notes-ie-llm`: Re-examine the clinical IE performance from section 01 from a benchmark perspective.

## References

1. Jin D, Pan E, Oufattole N, et al. "What Disease Does This Patient Have? A Large-Scale Open Domain Question Answering Dataset from Medical Exams (MedQA)." arXiv 2020. `https://arxiv.org/abs/2009.13081`
2. Jin Q, Dhingra B, Liu Z, et al. "PubMedQA: A Dataset for Biomedical Research Question Answering." EMNLP 2019.
3. Hendrycks D, Burns C, Basart S, et al. "Measuring Massive Multitask Language Understanding (MMLU)." ICLR 2021. `https://arxiv.org/abs/2009.03300`
4. Pal A, Umapathi LK, Sankarasubbu M. "MedMCQA: A Large-scale Multi-Subject Multi-Choice Dataset for Medical domain Question Answering." CHIL 2022.
5. OpenAI HealthBench (scheduled for release in 2025): `https://openai.com/index/healthbench/` (Please confirm the actual URL upon release).
6. Chen Z, Cano AH, Romanou A, et al. "Meditron-70B: Scaling Medical Pretraining for Large Language Models." EPFL 2023. `https://arxiv.org/abs/2311.16079`
7. Wu C, Zhang X, Zhang Y, et al. "PMC-LLaMA: Toward Building Open-source Language Models for Medicine." arXiv 2023.
8. Singhal K, Tu T, Gottweis J, et al. "Towards Expert-Level Medical Question Answering with Large Language Models (Med-PaLM 2)." Nature 2023. `https://www.nature.com/articles/s41586-023-06291-2`
9. Saab K, Tu T, Weng W-H, et al. "Capabilities of Gemini Models in Medicine (Med-Gemini)." Nature Medicine 2024. `https://www.nature.com/articles/s41591-024-03246-6`
10. OpenAI model benchmarks: `https://openai.com/index/gpt-4o-system-card/` · o1 · o3 system cards
11. Anthropic model benchmarks: `https://www.anthropic.com/news/claude-3-family` · Claude 4 system card
12. Xu R et al. "Benchmark Data Contamination of Large Language Models: A Survey." arXiv 2024. `https://arxiv.org/abs/2406.04244`
13. Lu Y, Bartolo M, Moore A, et al. "Fantastically Ordered Prompts and Where to Find Them." ACL 2022.
14. Zheng L, Chiang W-L, Sheng Y, et al. "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena." NeurIPS 2023.
15. Wei J, Wang X, Schuurmans D, et al. "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models." NeurIPS 2022.
16. HuggingFace datasets — bigbio/med_qa: `https://huggingface.co/datasets/bigbio/med_qa`
17. HuggingFace datasets — qiaojin/PubMedQA: `https://huggingface.co/datasets/qiaojin/PubMedQA`
18. HuggingFace datasets — cais/mmlu: `https://huggingface.co/datasets/cais/mmlu`
19. Meditron GitHub: `https://github.com/epfLLM/meditron`
20. HELM (Holistic Evaluation of Language Models): `https://crfm.stanford.edu/helm/`