{
  "metadata": {
    "updated": "2026-06-24T03:35:00Z",
    "description": "Aggregated results for multi-agent framework benchmark v2.2 and v2.3"
  },
  "versions": {
    "v2.2": {
      "comparison_id": "f3c892c1",
      "description": "Draft-then-validate orchestration fix",
      "questions": 20,
      "models": ["ollama/qwen2.5:7b", "gpt-4o"],
      "grader": "unified-gpt-4o",
      "results": {
        "ollama": {
          "B0_vanilla": { "avg_score": 2.6 },
          "B1_search": { "avg_score": 7.2 },
          "B2_search_reflection": { "avg_score": 8.0 },
          "MA_multi_agent": { "avg_score": 8.1, "convergence": "17/20 (85%)", "failures": [2, 8, 20] }
        },
        "gpt-4o": {
          "B0_vanilla": { "avg_score": 2.3 },
          "B1_search": { "avg_score": 8.2 },
          "B2_search_reflection": { "avg_score": 8.4 },
          "MA_multi_agent": { "avg_score": 7.8, "convergence": "19/20 (95%)", "failures": [20] }
        }
      },
      "key_finding": "Multi-agent orchestration compresses the capability gap between small and large models. MA on Ollama 7B (8.1) matches or beats MA on GPT-4o (7.8) and nearly matches B2 on GPT-4o (8.4)."
    },
    "v2.3": {
      "comparison_id": "0e156cf6",
      "description": "Deterministic grounding fix (per-field validator verdicts, controller-computed grounding)",
      "questions": 20,
      "models": ["ollama/qwen2.5:7b", "gpt-4o"],
      "grader": "unified-gpt-4o",
      "results": {
        "ollama": {
          "B0_vanilla": { "avg_score": 2.5 },
          "B1_search": { "avg_score": 7.5 },
          "B2_search_reflection": { "avg_score": 8.1 },
          "MA_multi_agent": { "avg_score": 8.0, "convergence": "18/20 (90%)", "failures": [15, 20] }
        },
        "gpt-4o": {
          "B0_vanilla": { "avg_score": 2.3 },
          "B1_search": { "avg_score": 7.8 },
          "B2_search_reflection": { "avg_score": 8.2 },
          "MA_multi_agent": { "avg_score": 7.9, "convergence": "19/20 (95%)", "failures": [20] }
        }
      },
      "key_finding": "Deterministic grounding improved convergence (Ollama 85%→90%, GPT-4o 95%→95%) without changing accuracy. MA remains competitive with B2. The headline finding holds: MA brings 7B model to frontier-model performance on evidence-grounded tasks."
    }
  },
  "headline_finding": "Multi-agent orchestration with proper grounding brings a 7B model (Ollama qwen2.5:7b, avg 8.0-8.1) to within 0.2-0.4 points of the best baseline on a frontier model (GPT-4o B2, avg 8.2-8.4) on evidence-grounded freshness questions. The overhead helps small models more than large models.",
  "cost_analysis": {
    "ollama_MA_per_question": { "llm_calls": 6, "duration_seconds": "60-80", "cost": "~$0 (local)" },
    "gpt4o_B2_per_question": { "llm_calls": 4, "duration_seconds": "15-20", "cost": "~$0.05-0.10 API" },
    "tradeoff": "~4-5x more wall-clock time and ~50% more tokens for ~100x lower cost and comparable accuracy"
  }
}