{
  "schema_version": "impact_mcp_search_benchmark_manifest_v1",
  "benchmark_id": "impact-mcp-search-comparison",
  "benchmark_version": "1.1.0",
  "purpose": "Compare the same LLM and prompt using direct search versus the public Impact MCP search workflow.",
  "prompt_count": 30,
  "repeats_per_prompt": 3,
  "lanes": {
    "direct_search": {
      "label": "Direct Search",
      "impact_mcp_allowed": false,
      "direct_web_search_allowed": true,
      "required_evidence": ["url", "title", "retrieved_at"],
      "boundary": "Use the harness's normal search capability. Do not call Impact MCP or use an Impact MCP result cached from another run."
    },
    "impact_mcp_search": {
      "label": "Impact MCP Search",
      "impact_mcp_allowed": true,
      "direct_web_search_allowed": false,
      "endpoint": "https://mcp.researchhub.foundation/mcp",
      "workflow": [
        "get_radar_snapshot_catalog",
        "search_trending_science",
        "get_trending_science_signal"
      ],
      "required_evidence": ["snapshot_version", "trend_id", "source_url", "retrieved_at"],
      "boundary": "Use only the public Impact MCP tools. Do not add direct web search, private records, or unapproved external sources."
    }
  },
  "metrics": [
    "relevance",
    "overall_search_quality",
    "recency",
    "traceability",
    "actionability"
  ],
  "use_cases": [
    {
      "slug": "research-intelligence",
      "title": "Research intelligence",
      "prompts": [
        {"id": "RI-01", "text": "Which public science conversations about gene therapy have recent attention? Return up to five candidate signals for an analyst to review, with concise reasons and source links."},
        {"id": "RI-02", "text": "Which emerging public signals about AI-assisted drug discovery should a research analyst investigate? Return up to five candidates with dates and source links."},
        {"id": "RI-03", "text": "What source-linked public coverage exists around radiopharmaceutical research and development? Return up to five recent signals and explain why each merits review."},
        {"id": "RI-04", "text": "What emerging public science conversations about antimicrobial resistance are receiving attention? Return up to five source-linked signals for further investigation."},
        {"id": "RI-05", "text": "Which public science signals about antibody therapeutics and immune engineering are worth reviewing now? Return up to five candidates, their observed dates, and evidence links."}
      ]
    },
    {
      "slug": "science-communications",
      "title": "Science communications",
      "prompts": [
        {"id": "SC-01", "text": "What is the recent public science-news context around mRNA vaccine research? Return up to five source-linked signals an editor should review before drafting an explainer."},
        {"id": "SC-02", "text": "What are the latest biomedical research news developments an editor should review before preparing a general-audience briefing? Return up to five source-linked signals with publication or observation dates."},
        {"id": "SC-03", "text": "Which public conversations about longevity research need clarification or follow-up reporting? Return up to five source-linked signals and identify the follow-up need."},
        {"id": "SC-04", "text": "What public coverage should be reviewed before preparing a general-audience explanation of gene editing? Return up to five recent signals with source links."},
        {"id": "SC-05", "text": "What public attention is developing around clinical artificial intelligence? Return up to five source-linked signals an editor should assess before describing the topic as emerging or widely discussed."}
      ]
    },
    {
      "slug": "funder-portfolio-context",
      "title": "Funder and portfolio context",
      "prompts": [
        {"id": "FP-01", "text": "What public science conversations provide context for a cancer-immunotherapy portfolio review? Return up to five source-linked signals and state that they are context, not funding recommendations."},
        {"id": "FP-02", "text": "Which public signals about AI for drug discovery are worth investigating before a portfolio planning discussion? Return up to five candidates with dates and sources."},
        {"id": "FP-03", "text": "What public attention is emerging around rare-disease research? Return up to five source-linked signals for a funder team to review alongside its own due diligence."},
        {"id": "FP-04", "text": "Which public science conversations about cell and gene therapy manufacturing could inform portfolio context? Return up to five signals with source links and evidence dates."},
        {"id": "FP-05", "text": "What research areas have emerging public coverage but require uncertainty or evidence follow-up before portfolio discussion? Return up to five source-linked candidates."}
      ]
    },
    {
      "slug": "policy-and-program-horizon-scanning",
      "title": "Policy and program horizon scanning",
      "prompts": [
        {"id": "PP-01", "text": "Which public science topics related to antimicrobial resistance should a program team investigate? Return up to five source-linked signals and avoid presenting them as policy recommendations."},
        {"id": "PP-02", "text": "What public conversations around antimicrobial resistance and antibiotic stewardship merit background research for a policy brief? Return up to five recent signals with dates and sources."},
        {"id": "PP-03", "text": "Which vaccine-research signals could seed a policy background brief? Return up to five source-linked candidates and identify what requires authoritative follow-up."},
        {"id": "PP-04", "text": "What emerging public discussions about clinical AI should a policy team monitor? Return up to five source-linked signals and separate attention from policy impact."},
        {"id": "PP-05", "text": "What source-linked public science material could seed a human-reviewed horizon scan for a research program? Return up to five candidates and explain the next review step."}
      ]
    },
    {
      "slug": "ai-research-agents",
      "title": "AI research agents",
      "prompts": [
        {"id": "AA-01", "text": "How can an agent search public Trending Science signals without scraping the website? Return the applicable workflow and any source-linked public signals needed to demonstrate it."},
        {"id": "AA-02", "text": "What should an agent do before making a claim about the current Science Radar? Return the required discovery and retrieval steps with their evidence."},
        {"id": "AA-03", "text": "Find public Trending Science signals for a topic and return a reproducible review queue with stable identifiers, dates, and source links."},
        {"id": "AA-04", "text": "What data does the public Impact MCP expose and what does it exclude? Return the answer with inspectable public evidence and any relevant current catalog information."},
        {"id": "AA-05", "text": "How should an agent search, retrieve, and review source-linked public Radar signals before writing a research summary? Return a bounded workflow and evidence."}
      ]
    },
    {
      "slug": "top-50-discovery",
      "title": "Top-50 discovery and ranking",
      "description": "Full Impact MCP long-list retrieval tasks. Use the public corpus for funders, projects, and programs; use the MCP's published biomedical signal collection for recent news/signals. Never access a frontend page. Each task requires an explicitly bounded ranked set, transparent inclusion criteria, and source-linked evidence. A result may contain fewer than 50 items when the lane cannot substantiate 50; it must never pad the list with invented or duplicate entries.",
      "fairness_contract": "Every lane receives the identical biomedical prompt and Top-50 target. Direct Search and Impact MCP may use different evidence identifiers, but neither lane receives hidden criteria or a manually authored answer key.",
      "mcp_workflow": {
        "T50-01": ["get_impact_catalog", "search_impact_records", "get_impact_record"],
        "T50-02": ["get_impact_catalog", "search_impact_records", "get_impact_record"],
        "T50-03": ["get_radar_snapshot_catalog", "search_trending_science", "get_trending_science_signal"],
        "T50-04": ["get_impact_catalog", "search_impact_records", "get_impact_record"],
        "T50-05": ["get_impact_catalog", "search_impact_records", "get_impact_record"]
      },
      "prompts": [
        {"id": "T50-01", "text": "Build a ranked Top 50 list of funders, foundations, or grant programs relevant to supporting research on antimicrobial resistance. For each item provide the name, why it is relevant, the strongest available source link, and the evidence date. Do not invent entries; return fewer than 50 if the lane cannot substantiate them."},
        {"id": "T50-02", "text": "Build a ranked Top 50 list of the latest publicly documented preclinical biomedical research updates. For each update provide the project or study title, development stage or preclinical context when available, source link, evidence date, and a concise reason it merits attention. Never pad the list with invented updates."},
        {"id": "T50-03", "text": "Build a ranked Top 50 list of recent news or public science signals about gene therapy. For each item provide the headline or signal title, publication or observation date, source link, and a one-sentence reason it belongs in the ranking. Do not claim completeness and do not invent missing sources."},
        {"id": "T50-04", "text": "Build a ranked Top 50 list of publicly documented biomedical research projects addressing gene therapy and translational therapeutics. For each project provide the title, project or award identifier when available, research evidence, source link, and date. Do not fill gaps with unsupported projects."},
        {"id": "T50-05", "text": "Build a ranked Top 50 list of active biomedical research programs, initiatives, or publicly documented projects addressing rare diseases and translational therapeutics. For each item provide the program name, geography or sponsor when available, source link, evidence date, and a concise relevance reason. Return fewer than 50 rather than inventing or duplicating programs."}
      ]
    }
  ],
  "run_contract": {
    "same_prompt_required": true,
    "same_model_within_pair_required": true,
    "same_settings_within_pair_required": true,
    "required_metadata": [
      "prompt_id",
      "prompt_text",
      "lane",
      "repeat_index",
      "harness_name",
      "harness_version",
      "model",
      "model_version",
      "system_instruction_version",
      "temperature",
      "max_output_tokens",
      "started_at",
      "completed_at",
      "timezone"
    ],
    "unavailable_result_policy": "Report status=unavailable with a reason. Never invent results, sources, dates, IDs, or scores.",
    "privacy_policy": "Do not include API keys, credentials, private data, raw captures, internal metadata, database values, or infrastructure secrets."
  },
  "judge": {
    "protocol": "absolute_rubric_plus_blind_pairwise",
    "score_scale": "0_to_4",
    "lane_label_hidden": true,
    "pair_order_randomized": true,
    "answer_key": null,
    "judge_model_required": true
  },
  "artifacts": {
    "instructions": "benchmark-instructions.md",
    "manifest": "benchmark-manifest.v1.json",
    "rubric": "judge-rubric.v1.json",
    "result_schema": "result-schema.v1.json",
    "examples_directory": "examples/"
  }
}