{
  "benchmark": "memq-bench-retrieval",
  "generatedAt": "2026-04-09T04:03:56.996Z",
  "corpus": {
    "caseCount": 12,
    "namespaces": [
      "apollo",
      "orion",
      "zephyr"
    ],
    "families": [
      "disambiguation",
      "freshness",
      "incident",
      "preference",
      "procedural",
      "project_memory",
      "runbook"
    ]
  },
  "providers": {
    "keyword_baseline": {
      "runs": 36,
      "completed": 36,
      "failed": 0,
      "skipped": 0,
      "avgLatencyMs": 0,
      "p95LatencyMs": 0,
      "hitAtKRate": 1,
      "primaryAt1Rate": 0.5833333333333334,
      "avgRecallAtK": 0.9166666666666666,
      "avgPrecisionAtK": 0.21666666666666679,
      "avgMrr": 0.861111111111111,
      "leakageFreeRate": 0.6666666666666666
    },
    "mem0_oss": {
      "runs": 36,
      "completed": 36,
      "failed": 0,
      "skipped": 0,
      "avgLatencyMs": 3381,
      "p95LatencyMs": 4163,
      "hitAtKRate": 1,
      "primaryAt1Rate": 0.5833333333333334,
      "avgRecallAtK": 1,
      "avgPrecisionAtK": 0.2500000000000001,
      "avgMrr": 0.9166666666666666,
      "leakageFreeRate": 0.6666666666666666
    },
    "memq_mcp": {
      "runs": 36,
      "completed": 36,
      "failed": 0,
      "skipped": 0,
      "avgLatencyMs": 21,
      "p95LatencyMs": 31,
      "hitAtKRate": 1,
      "primaryAt1Rate": 1,
      "avgRecallAtK": 1,
      "avgPrecisionAtK": 0.2500000000000001,
      "avgMrr": 1,
      "leakageFreeRate": 1
    }
  },
  "comparisons": [
    {
      "provider": "memq_mcp",
      "comparator": "mem0_oss",
      "primaryAt1DeltaPoints": 42,
      "recallDeltaPoints": 0,
      "latencyDeltaMs": -3360
    },
    {
      "provider": "memq_mcp",
      "comparator": "keyword_baseline",
      "primaryAt1DeltaPoints": 42,
      "recallDeltaPoints": 8,
      "latencyDeltaMs": 21
    }
  ],
  "reproducibility": {
    "resultCount": 108,
    "providers": [
      "keyword_baseline",
      "mem0_oss",
      "memq_mcp"
    ],
    "repetitions": [
      1,
      2,
      3
    ]
  },
  "artifacts": [
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-alex-billing-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-alex-billing-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-alex-billing-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-api-rotation-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-api-rotation-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-api-rotation-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-cfo-email-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-cfo-email-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-cfo-email-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-checkout-owner-deadline-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-checkout-owner-deadline-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-checkout-owner-deadline-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-incident-root-cause-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-incident-root-cause-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-incident-root-cause-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-memq-loop-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-memq-loop-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-memq-loop-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-notify-dana-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-notify-dana-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-notify-dana-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-pagerduty-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-pagerduty-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-pagerduty-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-postgres-failover-fix-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-postgres-failover-fix-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-apollo-postgres-failover-fix-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-orion-api-rotation-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-orion-api-rotation-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-orion-api-rotation-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-zephyr-pagerduty-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-zephyr-pagerduty-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-zephyr-pagerduty-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-zephyr-wallet-owner-deadline-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-zephyr-wallet-owner-deadline-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-keyword_baseline-zephyr-wallet-owner-deadline-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-alex-billing-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-alex-billing-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-alex-billing-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-api-rotation-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-api-rotation-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-api-rotation-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-cfo-email-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-cfo-email-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-cfo-email-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-checkout-owner-deadline-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-checkout-owner-deadline-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-checkout-owner-deadline-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-incident-root-cause-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-incident-root-cause-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-incident-root-cause-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-memq-loop-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-memq-loop-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-memq-loop-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-notify-dana-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-notify-dana-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-notify-dana-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-pagerduty-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-pagerduty-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-pagerduty-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-postgres-failover-fix-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-postgres-failover-fix-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-apollo-postgres-failover-fix-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-orion-api-rotation-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-orion-api-rotation-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-orion-api-rotation-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-zephyr-pagerduty-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-zephyr-pagerduty-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-zephyr-pagerduty-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-zephyr-wallet-owner-deadline-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-zephyr-wallet-owner-deadline-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-mem0_oss-zephyr-wallet-owner-deadline-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-alex-billing-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-alex-billing-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-alex-billing-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-api-rotation-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-api-rotation-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-api-rotation-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-cfo-email-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-cfo-email-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-cfo-email-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-checkout-owner-deadline-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-checkout-owner-deadline-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-checkout-owner-deadline-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-incident-root-cause-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-incident-root-cause-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-incident-root-cause-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-memq-loop-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-memq-loop-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-memq-loop-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-notify-dana-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-notify-dana-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-notify-dana-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-pagerduty-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-pagerduty-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-pagerduty-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-postgres-failover-fix-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-postgres-failover-fix-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-apollo-postgres-failover-fix-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-orion-api-rotation-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-orion-api-rotation-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-orion-api-rotation-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-zephyr-pagerduty-current-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-zephyr-pagerduty-current-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-zephyr-pagerduty-current-r3.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-zephyr-wallet-owner-deadline-r1.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-zephyr-wallet-owner-deadline-r2.json",
    "artifacts/retrieval-results-mcp-improved-v4/memq-mcp-server-improved-v4-memq_mcp-zephyr-wallet-owner-deadline-r3.json"
  ]
}