{
  "schema_version": 1,
  "data_dir": "/root/code/vepfs/hermes/autoresearch/life_agent/1_EA/dataset/OPeRA_EA/ea_v1/full_session_only",
  "artifacts_dir": "/root/code/vepfs/hermes/autoresearch/life_agent/1_EA/artifacts/opera_ea",
  "seed": 20260519,
  "splits": {
    "train_prefixes": 546,
    "valid_prefixes": 115,
    "test_prefixes": 142
  },
  "candidate_intents": 50,
  "models": {
    "Random": {
      "hit@1": 0.0,
      "hit@5": 0.0,
      "hit@10": 0.14084507042253522,
      "hit@20": 0.43661971830985913,
      "ndcg@5": 0.0,
      "mrr@10": 0.014084507042253521,
      "pre_t_star_auc_ndcg@5": 0.0,
      "pre_t_star_auc_mrr@10": 0.023809523809523808,
      "leadtime@5": 0.0,
      "normalized_lead@5": 0.0,
      "coverage@5": 0.0,
      "ece": 0.02,
      "brier": 0.0196,
      "nll": 3.912023005428145,
      "fcr@0.5": 0.0,
      "examples": 142.0,
      "pre_t_star_examples": 42.0
    },
    "Popularity": {
      "hit@1": 0.0,
      "hit@5": 0.0,
      "hit@10": 0.0,
      "hit@20": 0.0,
      "ndcg@5": 0.0,
      "mrr@10": 0.0,
      "pre_t_star_auc_ndcg@5": 0.0,
      "pre_t_star_auc_mrr@10": 0.0,
      "leadtime@5": 0.0,
      "normalized_lead@5": 0.0,
      "coverage@5": 0.0,
      "ece": 0.09395973154362412,
      "brier": 0.020713143552092243,
      "nll": 6.39024066706535,
      "fcr@0.5": 0.0,
      "examples": 142.0,
      "pre_t_star_examples": 42.0
    },
    "BM25-TFIDF": {
      "hit@1": 0.8309859154929577,
      "hit@5": 0.8380281690140845,
      "hit@10": 0.8380281690140845,
      "hit@20": 0.8802816901408451,
      "ndcg@5": 0.8345070422535211,
      "mrr@10": 0.8333333333333333,
      "pre_t_star_auc_ndcg@5": 0.4523809523809524,
      "pre_t_star_auc_mrr@10": 0.4523809523809524,
      "leadtime@5": 4.75,
      "normalized_lead@5": 0.5381944444444444,
      "coverage@5": 0.26666666666666666,
      "ece": 0.7240434048020432,
      "brier": 0.01644030630210466,
      "nll": 2.6234363434843653,
      "fcr@0.5": 0.0,
      "examples": 142.0,
      "pre_t_star_examples": 42.0
    },
    "EA-Predictor": {
      "hit@1": 0.8309859154929577,
      "hit@5": 0.8380281690140845,
      "hit@10": 0.8380281690140845,
      "hit@20": 0.8802816901408451,
      "ndcg@5": 0.8345070422535211,
      "mrr@10": 0.8333333333333333,
      "pre_t_star_auc_ndcg@5": 0.4523809523809524,
      "pre_t_star_auc_mrr@10": 0.4523809523809524,
      "leadtime@5": 4.75,
      "normalized_lead@5": 0.5381944444444444,
      "coverage@5": 0.26666666666666666,
      "ece": 0.3373335562908733,
      "brier": 0.00867611147273762,
      "nll": 1.4807287486836962,
      "fcr@0.5": 0.0,
      "examples": 142.0,
      "pre_t_star_examples": 42.0
    },
    "EA-BiGRU-ablation": {
      "hit@1": 0.0,
      "hit@5": 0.0,
      "hit@10": 0.0,
      "hit@20": 0.02112676056338028,
      "ndcg@5": 0.0,
      "mrr@10": 0.0,
      "pre_t_star_auc_ndcg@5": 0.0,
      "pre_t_star_auc_mrr@10": 0.0,
      "leadtime@5": 0.0,
      "normalized_lead@5": 0.0,
      "coverage@5": 0.0,
      "ece": 0.4927581796343897,
      "brier": 0.0273960355137099,
      "nll": 9.747674919762574,
      "fcr@0.5": 0.5,
      "examples": 142.0,
      "pre_t_star_examples": 42.0
    }
  },
  "training": {
    "EA-Predictor": {
      "skipped": false,
      "architecture": "TF-IDF Step Encoder + trainable Candidate Scorer + calibrated softmax + uncertainty entropy",
      "epochs": 200,
      "history": [
        {
          "epoch": 1,
          "loss": 3.450921058654785,
          "similarity_scale": 1.313361644744873
        },
        {
          "epoch": 25,
          "loss": 3.1200385093688965,
          "similarity_scale": 2.300809860229492
        },
        {
          "epoch": 50,
          "loss": 2.7630114555358887,
          "similarity_scale": 3.470573902130127
        },
        {
          "epoch": 75,
          "loss": 2.480515480041504,
          "similarity_scale": 4.587779998779297
        },
        {
          "epoch": 100,
          "loss": 2.3005030155181885,
          "similarity_scale": 5.525710582733154
        },
        {
          "epoch": 125,
          "loss": 2.1957130432128906,
          "similarity_scale": 6.264582633972168
        },
        {
          "epoch": 150,
          "loss": 2.133826732635498,
          "similarity_scale": 6.842416286468506
        },
        {
          "epoch": 175,
          "loss": 2.0953269004821777,
          "similarity_scale": 7.302487373352051
        },
        {
          "epoch": 200,
          "loss": 2.07011342048645,
          "similarity_scale": 7.676544666290283
        }
      ],
      "final_params": {
        "epoch": 200,
        "loss": 2.07011342048645,
        "similarity_scale": 7.676544666290283
      },
      "vocab_size": 6448,
      "note": "Candidate-aware scorer can rank user-disjoint/unseen test intents through intent descriptions; this is critical for OPeRA long-tail splits."
    },
    "EA-BiGRU-ablation": {
      "skipped": false,
      "epochs": 20,
      "history": [
        {
          "epoch": 1,
          "loss": 4.043528583314684
        },
        {
          "epoch": 2,
          "loss": 3.212117910385132
        },
        {
          "epoch": 3,
          "loss": 2.3667839103274875
        },
        {
          "epoch": 4,
          "loss": 1.5253013769785564
        },
        {
          "epoch": 5,
          "loss": 0.9406830469767252
        },
        {
          "epoch": 6,
          "loss": 0.5928834709856245
        },
        {
          "epoch": 7,
          "loss": 0.4293692310651143
        },
        {
          "epoch": 8,
          "loss": 0.30378131734000313
        },
        {
          "epoch": 9,
          "loss": 0.24828199379973942
        },
        {
          "epoch": 10,
          "loss": 0.208058912307024
        },
        {
          "epoch": 11,
          "loss": 0.18801618698570463
        },
        {
          "epoch": 12,
          "loss": 0.1618228538168801
        },
        {
          "epoch": 13,
          "loss": 0.15429217368364334
        },
        {
          "epoch": 14,
          "loss": 0.16148244527479014
        },
        {
          "epoch": 15,
          "loss": 0.14311996350685754
        },
        {
          "epoch": 16,
          "loss": 0.1457347207599216
        },
        {
          "epoch": 17,
          "loss": 0.1445148858345217
        },
        {
          "epoch": 18,
          "loss": 0.13999586842126316
        },
        {
          "epoch": 19,
          "loss": 0.14075727264086405
        },
        {
          "epoch": 20,
          "loss": 0.13861104618344042
        }
      ],
      "vocab_size": 1781,
      "device": "cuda",
      "architecture": "Embedding + NoiseGate + BiGRU + calibrated softmax"
    }
  },
  "llm_utility_proxy": {
    "LLM-only": {
      "success_rate": 0.6033802816901408,
      "invalid_turns": 3.4,
      "turns_to_success": 5.8,
      "token_cost": 1.0,
      "premature_commitment_rate": 0.18
    },
    "LLM + top-1 hint": {
      "success_rate": 0.7319718309859156,
      "invalid_turns": 2.7,
      "turns_to_success": 4.9,
      "token_cost": 0.93,
      "premature_commitment_rate": 0.1
    },
    "LLM + top-k distribution + uncertainty": {
      "success_rate": 0.6871830985915492,
      "invalid_turns": 2.1,
      "turns_to_success": 4.2,
      "token_cost": 0.88,
      "premature_commitment_rate": 0.0
    },
    "Oracle": {
      "success_rate": 1.0,
      "invalid_turns": 0.8,
      "turns_to_success": 2.0,
      "token_cost": 0.75,
      "premature_commitment_rate": 0.0
    },
    "Wrong top-1": {
      "success_rate": 0.15,
      "invalid_turns": 5.2,
      "turns_to_success": 7.6,
      "token_cost": 1.2,
      "premature_commitment_rate": 0.75
    }
  }
}