{
  "suite_id": "novel_feedback_v4_baseline_diagnosis",
  "created_utc": "2026-05-12T03:00:43+00:00",
  "v3_reproduction": {
    "suite_id": "novel_feedback_v3_suite",
    "experiment_count": 96,
    "scenario_count": 12,
    "method_count": 8,
    "state_vs_memory_uplift": 0.2748,
    "uncertainty_vs_memory_uplift": 0.3498,
    "qidian_dta_status": "Qidian-Webnovel reader comments/replies require a Data Transfer Agreement; this suite uses metadata-only/DTA-ready adapter paths with no bypass and no protected scraping."
  },
  "code_walk": {
    "state_conditioned_planning": {
      "input_sources": [
        "explicit feedback",
        "implicit dwell/fast-swipe/continue signals",
        "reader state",
        "story/content axes"
      ],
      "state_representation": [
        "trust",
        "fatigue",
        "created_expectations",
        "pacing_speed",
        "agency",
        "scope",
        "anti_scope",
        "ttl",
        "provenance"
      ],
      "update_source": "gold/simulated state diff derived from feedback event and content axis mismatch",
      "ttl_scope": "next_1_to_2_chapters or until_current_arc_resolution; not global unless story-world evidence exists",
      "applies_to": "direct_generation_prompt",
      "generation_effect": "planner/critic checklist and one-shot chapter plan are conditioned on scoped state diff; no candidate set or reranking is produced"
    },
    "uncertainty_aware_planning": {
      "input_sources": [
        "same as state_conditioned_planning",
        "confidence estimates",
        "ambiguity/noise indicators"
      ],
      "has_ask_hold_gate": true,
      "ask_hold_gate": "low-confidence or contradictory feedback can ask a clarification or hold a state update rather than overgeneralizing",
      "update_scope": "conservative when implicit-only/noisy/delayed signals disagree",
      "generation_effect": "direct generation prompt includes confidence-bounded constraints and avoids irreversible global changes"
    }
  },
  "worst_scenarios": [
    {
      "scenario": "adversarial_contradiction",
      "label": "Adversarial contradictory request",
      "severity_score": 0.8913,
      "failure_type": "calibration_uncertainty_error",
      "evidence": {
        "difficulty": 0.42,
        "noise": 0.34,
        "drift": 0.12,
        "uncertainty_aware_dynamic_return": 2.4712,
        "oracle_dynamic_return": 2.64,
        "oracle_gap": 0.1688,
        "raw_feedback_overgeneralization_rate": 0.5384,
        "memory_stale_use_rate": 0.3996,
        "dominant_failure_mode": "Without conservative scope, updates globalize local sarcasm into a lasting preference."
      },
      "v4_fix": "add ordinal/confidence labels, ECE, ask/hold gate and contradiction stress tests"
    },
    {
      "scenario": "long_horizon_fatigue",
      "label": "Long-horizon fatigue accumulation",
      "severity_score": 0.814,
      "failure_type": "long_horizon_credit_assignment",
      "evidence": {
        "difficulty": 0.4,
        "noise": 0.18,
        "drift": 0.26,
        "uncertainty_aware_dynamic_return": 2.4784,
        "oracle_dynamic_return": 2.64,
        "oracle_gap": 0.1616,
        "raw_feedback_overgeneralization_rate": 0.384,
        "memory_stale_use_rate": 0.44,
        "dominant_failure_mode": "Methods that optimize immediate rating underweight trust loss and future churn."
      },
      "v4_fix": "evaluate feedback-to-future trace, delayed credit MRR and rollback scar rate"
    },
    {
      "scenario": "rollback_recovery",
      "label": "Rollback after correction",
      "severity_score": 0.773,
      "failure_type": "long_horizon_credit_assignment",
      "evidence": {
        "difficulty": 0.36,
        "noise": 0.2,
        "drift": 0.18,
        "uncertainty_aware_dynamic_return": 2.5032,
        "oracle_dynamic_return": 2.6532,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3832,
        "memory_stale_use_rate": 0.4128,
        "dominant_failure_mode": "State scars remain when previous updates lack TTL/provenance."
      },
      "v4_fix": "evaluate feedback-to-future trace, delayed credit MRR and rollback scar rate"
    }
  ],
  "all_scenario_diagnostics": [
    {
      "scenario": "baseline_clean",
      "label": "Clean multi-turn feedback",
      "severity_score": 0.5466,
      "failure_type": "generation_quality_issue",
      "evidence": {
        "difficulty": 0.08,
        "noise": 0.0,
        "drift": 0.02,
        "uncertainty_aware_dynamic_return": 2.6592,
        "oracle_dynamic_return": 2.8092,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3096,
        "memory_stale_use_rate": 0.3424,
        "dominant_failure_mode": "Remaining errors are mostly fine-grained expectation timing rather than preference-axis confusion."
      },
      "v4_fix": "add story bible consistency, character voice and plot continuity probes"
    },
    {
      "scenario": "implicit_only",
      "label": "Implicit-only reading signals",
      "severity_score": 0.6353,
      "failure_type": "state_tracking_error",
      "evidence": {
        "difficulty": 0.24,
        "noise": 0.18,
        "drift": 0.04,
        "uncertainty_aware_dynamic_return": 2.6074,
        "oracle_dynamic_return": 2.7224,
        "oracle_gap": 0.115,
        "raw_feedback_overgeneralization_rate": 0.3648,
        "memory_stale_use_rate": 0.3612,
        "dominant_failure_mode": "Dwell ambiguity: slow reading can mean delight, confusion, or fatigue."
      },
      "v4_fix": "train sequence encoder on history-window + state-delta labels and evaluate state_delta_cosine / adaptation half-life"
    },
    {
      "scenario": "explicit_only",
      "label": "Explicit-only comments and ratings",
      "severity_score": 0.6114,
      "failure_type": "generation_quality_issue",
      "evidence": {
        "difficulty": 0.16,
        "noise": 0.08,
        "drift": 0.03,
        "uncertainty_aware_dynamic_return": 2.6174,
        "oracle_dynamic_return": 2.7674,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3352,
        "memory_stale_use_rate": 0.3518,
        "dominant_failure_mode": "Sparse comments miss silent disengagement and late fatigue."
      },
      "v4_fix": "add story bible consistency, character voice and plot continuity probes"
    },
    {
      "scenario": "delayed_feedback_2_turn",
      "label": "Delayed feedback by two turns",
      "severity_score": 0.7355,
      "failure_type": "long_horizon_credit_assignment",
      "evidence": {
        "difficulty": 0.34,
        "noise": 0.15,
        "drift": 0.08,
        "uncertainty_aware_dynamic_return": 2.536,
        "oracle_dynamic_return": 2.686,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3708,
        "memory_stale_use_rate": 0.3812,
        "dominant_failure_mode": "Credit assignment can over-attribute frustration to the most recent chapter."
      },
      "v4_fix": "evaluate feedback-to-future trace, delayed credit MRR and rollback scar rate"
    },
    {
      "scenario": "noisy_feedback",
      "label": "Noisy and inconsistent feedback",
      "severity_score": 0.7392,
      "failure_type": "calibration_uncertainty_error",
      "evidence": {
        "difficulty": 0.3,
        "noise": 0.28,
        "drift": 0.05,
        "uncertainty_aware_dynamic_return": 2.5342,
        "oracle_dynamic_return": 2.6842,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.392,
        "memory_stale_use_rate": 0.369,
        "dominant_failure_mode": "Raw-feedback methods chase contradictory surface phrases."
      },
      "v4_fix": "add ordinal/confidence labels, ECE, ask/hold gate and contradiction stress tests"
    },
    {
      "scenario": "cold_start_users",
      "label": "Cold-start reader priors",
      "severity_score": 0.6865,
      "failure_type": "state_tracking_error",
      "evidence": {
        "difficulty": 0.27,
        "noise": 0.12,
        "drift": 0.06,
        "uncertainty_aware_dynamic_return": 2.5682,
        "oracle_dynamic_return": 2.7182,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3564,
        "memory_stale_use_rate": 0.3696,
        "dominant_failure_mode": "Prototype priors help but can overshoot when the first visible feedback contradicts the trope cluster."
      },
      "v4_fix": "train sequence encoder on history-window + state-delta labels and evaluate state_delta_cosine / adaptation half-life"
    },
    {
      "scenario": "preference_drift",
      "label": "Preference drift inside an arc",
      "severity_score": 0.7426,
      "failure_type": "state_tracking_error",
      "evidence": {
        "difficulty": 0.38,
        "noise": 0.16,
        "drift": 0.3,
        "uncertainty_aware_dynamic_return": 2.5262,
        "oracle_dynamic_return": 2.64,
        "oracle_gap": 0.1138,
        "raw_feedback_overgeneralization_rate": 0.3776,
        "memory_stale_use_rate": 0.4504,
        "dominant_failure_mode": "Static profile baselines lag behind newly created expectations."
      },
      "v4_fix": "train sequence encoder on history-window + state-delta labels and evaluate state_delta_cosine / adaptation half-life"
    },
    {
      "scenario": "adversarial_contradiction",
      "label": "Adversarial contradictory request",
      "severity_score": 0.8913,
      "failure_type": "calibration_uncertainty_error",
      "evidence": {
        "difficulty": 0.42,
        "noise": 0.34,
        "drift": 0.12,
        "uncertainty_aware_dynamic_return": 2.4712,
        "oracle_dynamic_return": 2.64,
        "oracle_gap": 0.1688,
        "raw_feedback_overgeneralization_rate": 0.5384,
        "memory_stale_use_rate": 0.3996,
        "dominant_failure_mode": "Without conservative scope, updates globalize local sarcasm into a lasting preference."
      },
      "v4_fix": "add ordinal/confidence labels, ECE, ask/hold gate and contradiction stress tests"
    },
    {
      "scenario": "long_horizon_fatigue",
      "label": "Long-horizon fatigue accumulation",
      "severity_score": 0.814,
      "failure_type": "long_horizon_credit_assignment",
      "evidence": {
        "difficulty": 0.4,
        "noise": 0.18,
        "drift": 0.26,
        "uncertainty_aware_dynamic_return": 2.4784,
        "oracle_dynamic_return": 2.64,
        "oracle_gap": 0.1616,
        "raw_feedback_overgeneralization_rate": 0.384,
        "memory_stale_use_rate": 0.44,
        "dominant_failure_mode": "Methods that optimize immediate rating underweight trust loss and future churn."
      },
      "v4_fix": "evaluate feedback-to-future trace, delayed credit MRR and rollback scar rate"
    },
    {
      "scenario": "rollback_recovery",
      "label": "Rollback after correction",
      "severity_score": 0.773,
      "failure_type": "long_horizon_credit_assignment",
      "evidence": {
        "difficulty": 0.36,
        "noise": 0.2,
        "drift": 0.18,
        "uncertainty_aware_dynamic_return": 2.5032,
        "oracle_dynamic_return": 2.6532,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3832,
        "memory_stale_use_rate": 0.4128,
        "dominant_failure_mode": "State scars remain when previous updates lack TTL/provenance."
      },
      "v4_fix": "evaluate feedback-to-future trace, delayed credit MRR and rollback scar rate"
    },
    {
      "scenario": "cross_user_contamination",
      "label": "Cross-user contamination guard",
      "severity_score": 0.6853,
      "failure_type": "privacy_contamination_error",
      "evidence": {
        "difficulty": 0.31,
        "noise": 0.14,
        "drift": 0.09,
        "uncertainty_aware_dynamic_return": 2.571,
        "oracle_dynamic_return": 2.696,
        "oracle_gap": 0.125,
        "raw_feedback_overgeneralization_rate": 0.3652,
        "memory_stale_use_rate": 0.3818,
        "dominant_failure_mode": "Naive shared memory writes contaminate the next user session."
      },
      "v4_fix": "separate user_memory vs story_state namespaces and test user-swap leakage"
    },
    {
      "scenario": "counterfactual_swap",
      "label": "Counterfactual user/content swaps",
      "severity_score": 0.7273,
      "failure_type": "generation_quality_issue",
      "evidence": {
        "difficulty": 0.33,
        "noise": 0.12,
        "drift": 0.11,
        "uncertainty_aware_dynamic_return": 2.5388,
        "oracle_dynamic_return": 2.6888,
        "oracle_gap": 0.15,
        "raw_feedback_overgeneralization_rate": 0.3636,
        "memory_stale_use_rate": 0.3894,
        "dominant_failure_mode": "Weak simulators collapse toward generic sentiment rather than user-conditioned response."
      },
      "v4_fix": "add story bible consistency, character voice and plot continuity probes"
    }
  ]
}