{
  "suite_id": "novel_feedback_v3_suite",
  "title": "Novel Feedback v3 Experiment Suite",
  "created_utc": "2026-05-11T19:52:57+00:00",
  "uses_reranking": false,
  "direct_generation_policy": "state-conditioned direct generation",
  "experiment_count": 96,
  "scenario_count": 12,
  "method_count": 8,
  "data_condition_count": 4,
  "simulator_variant_count": 4,
  "qidian_dta_status": "Qidian-Webnovel reader comments/replies require a Data Transfer Agreement; this suite uses metadata-only/DTA-ready adapter paths with no bypass and no protected scraping.",
  "data_sources": [
    {
      "id": "public_text_augmented_feedback",
      "description": "WebNovelBench/NovelUpdates public snapshots plus calibrated feedback trajectories.",
      "license_class": "public_or_metadata_review_required"
    },
    {
      "id": "qidian_metadata_dta_ready",
      "description": "Qidian/WebNovel metadata mapping and DTA-ready adapter path; no protected comments are copied or scraped.",
      "license_class": "metadata_only_dta_required_for_reader_response"
    },
    {
      "id": "implicit_action_augmented",
      "description": "Implicit dwell/fast-swipe/reread/continue perturbations for ambiguity stress tests.",
      "license_class": "derived_from_public_snapshot"
    },
    {
      "id": "counterfactual_user_swap",
      "description": "Synthetic counterfactual swaps of user archetype and content axes for simulator validity tests.",
      "license_class": "synthetic_from_public_features"
    }
  ],
  "simulator_variants": [
    {
      "id": "rf_dynamic_simulator_v1",
      "description": "Existing RandomForest dynamic simulator smoke baseline anchored to v1 metrics.",
      "compute": "cpu_sklearn"
    },
    {
      "id": "ordinal_transition_baseline",
      "description": "Ordinal logistic/ridge transition baseline for interpretable rating and state deltas.",
      "compute": "cpu_sklearn"
    },
    {
      "id": "sequence_state_cpu",
      "description": "CPU sequence-state simulator approximation with history-window features.",
      "compute": "cpu_numpy"
    },
    {
      "id": "uncertainty_calibrated_cpu",
      "description": "Calibrated ensemble-style simulator with confidence intervals and abstention gates.",
      "compute": "cpu_sklearn"
    }
  ],
  "methods": [
    {
      "id": "no_update",
      "label": "No update",
      "note": "Frozen agent state; useful lower bound for dynamic personalization."
    },
    {
      "id": "static_profile",
      "label": "Static profile prompt",
      "note": "Uses a fixed reader archetype but does not model text-induced state transitions."
    },
    {
      "id": "memory_only",
      "label": "Summary memory only",
      "note": "Summarizes feedback but lacks scoped planner/critic/generator routing."
    },
    {
      "id": "raw_feedback_rag",
      "label": "Raw feedback RAG",
      "note": "Retrieves raw feedback snippets; strong on clean comments but fragile to noise/sarcasm."
    },
    {
      "id": "state_conditioned_planning",
      "label": "State-conditioned planning",
      "note": "Directly conditions the next chapter plan and prompt on scoped user-state diffs."
    },
    {
      "id": "uncertainty_aware_planning",
      "label": "Uncertainty-aware planning",
      "note": "Adds confidence, ask/hold gates, and ambiguity-aware direct generation prompts."
    },
    {
      "id": "conservative_state_update",
      "label": "Conservative scoped update",
      "note": "Adds TTL/provenance/scope gates and rollback-friendly state diffs."
    },
    {
      "id": "oracle_update",
      "label": "Oracle update",
      "note": "Upper bound using hidden gold state transition; not available to the agent."
    }
  ],
  "scenario_ids": [
    "baseline_clean",
    "implicit_only",
    "explicit_only",
    "delayed_feedback_2_turn",
    "noisy_feedback",
    "cold_start_users",
    "preference_drift",
    "adversarial_contradiction",
    "long_horizon_fatigue",
    "rollback_recovery",
    "cross_user_contamination",
    "counterfactual_swap"
  ],
  "anchor_dynamic_simulator_v1": {
    "simulator_metrics_v1": {
      "n_events": 360,
      "train_events": 234,
      "test_events": 126,
      "rating_mae": 0.172,
      "continue_auc": 0.9258,
      "fast_swipe_auc": 0.9879,
      "transition_trust_mae": 0.0134,
      "transition_fatigue_mae": 0.0,
      "calibration_ece": 0.0849,
      "counterfactual_separation": 0.1333
    },
    "closed_loop_v1": {
      "uses_reranking": false,
      "methods": {
        "no_update": {
          "dynamic_return": 2.3827,
          "expectation_alignment": 0.0,
          "personalization_diversity": 0.225,
          "state_growth": 0.0
        },
        "static_profile": {
          "dynamic_return": 2.4816,
          "expectation_alignment": 0.3333,
          "personalization_diversity": 0.2948,
          "state_growth": 0.667
        },
        "memory_only": {
          "dynamic_return": 2.4216,
          "expectation_alignment": 0.2667,
          "personalization_diversity": 0.2202,
          "state_growth": 0.5
        },
        "state_conditioned_planning": {
          "dynamic_return": 2.5917,
          "expectation_alignment": 0.7667,
          "personalization_diversity": 0.277,
          "state_growth": 1.833
        },
        "oracle_update": {
          "dynamic_return": 2.7894,
          "expectation_alignment": 0.8,
          "personalization_diversity": 0.485,
          "state_growth": 3.833
        }
      }
    }
  },
  "gpu_path": {
    "status": "ready_not_submitted",
    "reason_not_submitted": "CPU suite is sufficient for deterministic artifact generation; GPU should be used for the next sequence/encoder simulator training run, not for this synthetic matrix expansion.",
    "provider": "VolcEngine ML Platform",
    "workspace": "/root/code/vepfs/hermes/gpu",
    "recommended_task": {
      "name": "novel_feedback_v3_sequence_simulator",
      "entrypoint": "train_sequence_state_simulator.py --suite v3_suite --model transformer_small --eval closed_loop",
      "queue": "q-20251224104457-jzdpj",
      "creator_filter": "252306041",
      "expected_outputs": [
        "sequence_simulator_metrics.json",
        "sim_to_real_calibration_stub.json",
        "closed_loop_gpu_comparison.json"
      ]
    },
    "safety": "Use volc ml_task submit -c <task-specific>.yaml; keep credentials/YAML/logs inside /root/code/vepfs/hermes/gpu; do not persist AK/SK."
  },
  "artifact_index": {
    "experiment_matrix": "experiment_matrix.json",
    "metric_definitions": "metric_definitions.json",
    "scenario_results": "scenario_results.json",
    "method_comparison": "method_comparison.json",
    "stress_results": "stress_results.json",
    "counterfactual_heatmap": "counterfactual_heatmap.json",
    "rollback_recovery": "rollback_recovery.json",
    "contamination_checks": "contamination_checks.json",
    "direct_generation_samples": "direct_generation_samples.jsonl",
    "visualization": "visualization.html",
    "report": "suite_report.md",
    "gpu_path": "gpu_path.json"
  }
}