{
  "suite_id": "novel_feedback_v3_suite",
  "uses_reranking": false,
  "average_by_method": {
    "no_update": {
      "dynamic_return": 2.2262,
      "expectation_alignment": 0.002,
      "ncfu": 0.0,
      "selectivity": 0.0,
      "regret_to_oracle": 0.6738,
      "overgeneralization_rate": 0.1284,
      "stale_memory_use_rate": 0.5774,
      "calibration_ece": 0.1819,
      "counterfactual_separation": 0.0713,
      "rollback_success": 0.0981,
      "behavioral_scar_rate": 0.3894,
      "contamination_leakage": 0.0,
      "direct_generation_diversity": 0.168,
      "robustness_score": 0.1666,
      "no_reranking_compliance": 1.0,
      "label": "No update",
      "note": "Frozen agent state; useful lower bound for dynamic personalization."
    },
    "static_profile": {
      "dynamic_return": 2.2737,
      "expectation_alignment": 0.1869,
      "ncfu": 0.1103,
      "selectivity": 0.0645,
      "regret_to_oracle": 0.6263,
      "overgeneralization_rate": 0.2284,
      "stale_memory_use_rate": 0.4374,
      "calibration_ece": 0.1679,
      "counterfactual_separation": 0.0983,
      "rollback_success": 0.2115,
      "behavioral_scar_rate": 0.2994,
      "contamination_leakage": 0.01,
      "direct_generation_diversity": 0.248,
      "robustness_score": 0.3066,
      "no_reranking_compliance": 1.0,
      "label": "Static profile prompt",
      "note": "Uses a fixed reader archetype but does not model text-induced state transitions."
    },
    "memory_only": {
      "dynamic_return": 2.2011,
      "expectation_alignment": 0.1469,
      "ncfu": 0.0703,
      "selectivity": 0.0199,
      "regret_to_oracle": 0.6989,
      "overgeneralization_rate": 0.2884,
      "stale_memory_use_rate": 0.3874,
      "calibration_ece": 0.1709,
      "counterfactual_separation": 0.0943,
      "rollback_success": 0.1947,
      "behavioral_scar_rate": 0.3394,
      "contamination_leakage": 0.0367,
      "direct_generation_diversity": 0.208,
      "robustness_score": 0.2767,
      "no_reranking_compliance": 1.0,
      "label": "Summary memory only",
      "note": "Summarizes feedback but lacks scoped planner/critic/generator routing."
    },
    "raw_feedback_rag": {
      "dynamic_return": 2.2479,
      "expectation_alignment": 0.2824,
      "ncfu": 0.1851,
      "selectivity": 0.0851,
      "regret_to_oracle": 0.6521,
      "overgeneralization_rate": 0.3784,
      "stale_memory_use_rate": 0.3274,
      "calibration_ece": 0.1659,
      "counterfactual_separation": 0.1083,
      "rollback_success": 0.2535,
      "behavioral_scar_rate": 0.4294,
      "contamination_leakage": 0.0567,
      "direct_generation_diversity": 0.2865,
      "robustness_score": 0.3225,
      "no_reranking_compliance": 1.0,
      "label": "Raw feedback RAG",
      "note": "Retrieves raw feedback snippets; strong on clean comments but fragile to noise/sarcasm."
    },
    "state_conditioned_planning": {
      "dynamic_return": 2.4759,
      "expectation_alignment": 0.6091,
      "ncfu": 0.4729,
      "selectivity": 0.5,
      "regret_to_oracle": 0.4241,
      "overgeneralization_rate": 0.1984,
      "stale_memory_use_rate": 0.1774,
      "calibration_ece": 0.1439,
      "counterfactual_separation": 0.1403,
      "rollback_success": 0.3879,
      "behavioral_scar_rate": 0.2094,
      "contamination_leakage": 0.025,
      "direct_generation_diversity": 0.3488,
      "robustness_score": 0.5487,
      "no_reranking_compliance": 1.0,
      "label": "State-conditioned planning",
      "note": "Directly conditions the next chapter plan and prompt on scoped user-state diffs."
    },
    "uncertainty_aware_planning": {
      "dynamic_return": 2.5509,
      "expectation_alignment": 0.6621,
      "ncfu": 0.5564,
      "selectivity": 0.5915,
      "regret_to_oracle": 0.3491,
      "overgeneralization_rate": 0.1284,
      "stale_memory_use_rate": 0.1474,
      "calibration_ece": 0.0999,
      "counterfactual_separation": 0.1453,
      "rollback_success": 0.4089,
      "behavioral_scar_rate": 0.1694,
      "contamination_leakage": 0.018,
      "direct_generation_diversity": 0.3698,
      "robustness_score": 0.6415,
      "no_reranking_compliance": 1.0,
      "label": "Uncertainty-aware planning",
      "note": "Adds confidence, ask/hold gates, and ambiguity-aware direct generation prompts."
    },
    "conservative_state_update": {
      "dynamic_return": 2.5034,
      "expectation_alignment": 0.6036,
      "ncfu": 0.5082,
      "selectivity": 0.5523,
      "regret_to_oracle": 0.3966,
      "overgeneralization_rate": 0.0934,
      "stale_memory_use_rate": 0.1674,
      "calibration_ece": 0.1019,
      "counterfactual_separation": 0.1393,
      "rollback_success": 0.5637,
      "behavioral_scar_rate": 0.0694,
      "contamination_leakage": 0.012,
      "direct_generation_diversity": 0.3303,
      "robustness_score": 0.6228,
      "no_reranking_compliance": 1.0,
      "label": "Conservative scoped update",
      "note": "Adds TTL/provenance/scope gates and rollback-friendly state diffs."
    },
    "oracle_update": {
      "dynamic_return": 2.6955,
      "expectation_alignment": 0.756,
      "ncfu": 0.8908,
      "selectivity": 0.7074,
      "regret_to_oracle": 0.2046,
      "overgeneralization_rate": 0.0884,
      "stale_memory_use_rate": 0.0574,
      "calibration_ece": 0.0295,
      "counterfactual_separation": 0.1543,
      "rollback_success": 0.6867,
      "behavioral_scar_rate": 0.0794,
      "contamination_leakage": 0.0,
      "direct_generation_diversity": 0.468,
      "robustness_score": 0.8207,
      "no_reranking_compliance": 1.0,
      "label": "Oracle update",
      "note": "Upper bound using hidden gold state transition; not available to the agent."
    }
  },
  "ranking_by_dynamic_return": [
    {
      "method": "oracle_update",
      "label": "Oracle update",
      "dynamic_return": 2.6955,
      "expectation_alignment": 0.756,
      "robustness_score": 0.8207
    },
    {
      "method": "uncertainty_aware_planning",
      "label": "Uncertainty-aware planning",
      "dynamic_return": 2.5509,
      "expectation_alignment": 0.6621,
      "robustness_score": 0.6415
    },
    {
      "method": "conservative_state_update",
      "label": "Conservative scoped update",
      "dynamic_return": 2.5034,
      "expectation_alignment": 0.6036,
      "robustness_score": 0.6228
    },
    {
      "method": "state_conditioned_planning",
      "label": "State-conditioned planning",
      "dynamic_return": 2.4759,
      "expectation_alignment": 0.6091,
      "robustness_score": 0.5487
    },
    {
      "method": "static_profile",
      "label": "Static profile prompt",
      "dynamic_return": 2.2737,
      "expectation_alignment": 0.1869,
      "robustness_score": 0.3066
    },
    {
      "method": "raw_feedback_rag",
      "label": "Raw feedback RAG",
      "dynamic_return": 2.2479,
      "expectation_alignment": 0.2824,
      "robustness_score": 0.3225
    },
    {
      "method": "no_update",
      "label": "No update",
      "dynamic_return": 2.2262,
      "expectation_alignment": 0.002,
      "robustness_score": 0.1666
    },
    {
      "method": "memory_only",
      "label": "Summary memory only",
      "dynamic_return": 2.2011,
      "expectation_alignment": 0.1469,
      "robustness_score": 0.2767
    }
  ],
  "primary_findings": [
    "Direct state-conditioned generation beats memory-only on average dynamic_return and expectation_alignment.",
    "Uncertainty-aware planning is strongest non-oracle method under noisy, delayed, drift and implicit-only conditions.",
    "Conservative scoped updates reduce overgeneralization, rollback scars and cross-user leakage.",
    "Raw feedback RAG is useful in clean settings but brittle under contradictory comments and rollback."
  ]
}