"""
Embedding A/B Experiment — ModuleDesk AI Draft quality.

Tests whether better embeddings (richer text, larger model) would materially
improve RAG recall for the eval ticket set, compared to the current production
strategy of embedding a 1-2 sentence gpt-4o-mini summary.

STRATEGIES
----------
S0 (baseline)  : query embedded on inbound question, corpus on summary_text,
                 text-embedding-3-small @ 512 dims  [current production]
S1 (full-text) : query embedded on inbound question, corpus on
                 "question + seller reply" full thread text,
                 text-embedding-3-small @ 512 dims  [same model, richer text]
S2 (large-dim) : same full-thread-text corpus, but embedded with
                 text-embedding-3-small @ 1536 dims (or text-embedding-3-large)
                 [bigger model/dims on richer text]

All scratch indexes are built in-memory — NO production embeddings touched.
For each eval ticket the same LLM relevance judge as eval_rag_quality.py is
used to assess whether the top-k retrieved precedents are genuinely relevant.

RECALL@3
--------
A ticket "has a precedent under strategy Sx" if at least ONE of the top-3
returned threads is judged relevant by gpt-4o-mini.

OUTPUT
------
Recall table + verdict printed to stdout.
Results JSON appended to tmp/eval-embedding-ab.json.
tmp/eval-draft-progress.md updated with a dated entry.

Usage:
    venv/bin/python scripts/eval_embedding_ab.py [--n 13] [--schema tenant_internal]
    venv/bin/python scripts/eval_embedding_ab.py --n 13 --schema tenant_internal
"""

from __future__ import annotations

import argparse
import json
import os
import sys
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

# ---------------------------------------------------------------------------
# Pricing
# ---------------------------------------------------------------------------
USD_EUR = 0.92
PRICE_USD_PER_1M: Dict[str, Dict[str, float]] = {
    "text-embedding-3-small":   {"in": 0.020, "out": 0.00},
    "text-embedding-3-large":   {"in": 0.130, "out": 0.00},
    "gpt-4o-mini":              {"in": 0.150, "out": 0.60},
}

def tokens_to_eur(model: str, prompt_tokens: int, completion_tokens: int = 0) -> float:
    p = PRICE_USD_PER_1M.get(model, {"in": 0.0, "out": 0.0})
    usd = (prompt_tokens * p["in"] + completion_tokens * p["out"]) / 1_000_000
    return usd * USD_EUR


@dataclass
class Costs:
    calls: List[Dict] = field(default_factory=list)

    def record(self, call_type: str, model: str, tokens: int, completion: int = 0) -> float:
        cost = tokens_to_eur(model, tokens, completion)
        self.calls.append({"type": call_type, "model": model, "tokens": tokens, "cost": cost})
        return cost

    def total(self) -> float:
        return sum(c["cost"] for c in self.calls)

    def breakdown(self) -> Dict[str, float]:
        agg: Dict[str, float] = {}
        for c in self.calls:
            key = f"{c['type']}:{c['model']}"
            agg[key] = agg.get(key, 0.0) + c["cost"]
        return agg


# ---------------------------------------------------------------------------
# Relevance judge (same as eval_rag_quality.py)
# ---------------------------------------------------------------------------

RELEVANCE_JUDGE_SYSTEM = """\
You are an expert support ticket analyst. You will be given two customer support questions:
- Question A: a new (incoming) support question
- Question B: a previously-resolved support question from the knowledge base

Your task: determine whether Question B is GENUINELY RELEVANT to Question A.
Genuinely relevant means: if the answer to B were shown to the agent, it would materially
help them answer A (same problem, same module feature, same error, same workflow).

Answer with JSON ONLY:
{
  "relevant": true or false,
  "confidence": 0-100 (how confident you are),
  "reason": "one sentence"
}

Be strict. A vague thematic overlap is NOT relevant. Relevance requires the same concrete problem.
"""


def judge_relevance(
    question_a: str,
    question_b: str,
    client,
    costs: Costs,
) -> Dict[str, Any]:
    """Ask gpt-4o-mini whether question_b is relevant to question_a."""
    user_content = (
        f"Question A (new ticket):\n---\n{question_a[:1000]}\n---\n\n"
        f"Question B (precedent):\n---\n{question_b[:1000]}\n---\n\n"
        "Respond with JSON only."
    )
    import openai as _openai
    try:
        resp = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": RELEVANCE_JUDGE_SYSTEM},
                {"role": "user", "content": user_content},
            ],
            temperature=0.0,
            max_tokens=150,
            response_format={"type": "json_object"},
        )
        usage = resp.usage
        if usage:
            costs.record("judge", "gpt-4o-mini", usage.prompt_tokens, usage.completion_tokens)
        result = json.loads(resp.choices[0].message.content)
        return {
            "relevant": bool(result.get("relevant", False)),
            "confidence": int(result.get("confidence", 0)),
            "reason": str(result.get("reason", "")),
        }
    except Exception as exc:
        return {"relevant": False, "confidence": 0, "reason": f"Judge failed: {exc}"}


# ---------------------------------------------------------------------------
# Embedding helpers
# ---------------------------------------------------------------------------

def embed_texts(
    client,
    texts: List[str],
    model: str,
    dims: int,
    costs: Costs,
    call_type: str = "embed",
) -> np.ndarray:
    """Embed a list of texts, return (N, dims) float32 array.
    Batches in chunks of 100 to stay within API limits."""
    all_vecs: List[List[float]] = []
    BATCH = 100
    for i in range(0, len(texts), BATCH):
        chunk = [t[:8000] for t in texts[i:i + BATCH]]
        resp = client.embeddings.create(input=chunk, model=model, dimensions=dims)
        if resp.usage:
            costs.record(call_type, model, resp.usage.prompt_tokens)
        sorted_data = sorted(resp.data, key=lambda d: d.index)
        all_vecs.extend([d.embedding for d in sorted_data])
    return np.array(all_vecs, dtype=np.float32)


def cosine_top_k(
    query_vec: np.ndarray,
    corpus_matrix_normed: np.ndarray,
    corpus_tids: List[int],
    exclude_tids: Optional[set] = None,
    top_k: int = 3,
) -> List[Tuple[int, float]]:
    """Return top_k (thread_id, similarity) pairs, excluding exclude_tids."""
    q = query_vec / (np.linalg.norm(query_vec) + 1e-10)
    sims = corpus_matrix_normed @ q
    ranked = np.argsort(sims)[::-1]
    results = []
    for idx in ranked:
        tid = corpus_tids[idx]
        if exclude_tids and tid in exclude_tids:
            continue
        results.append((tid, float(sims[idx])))
        if len(results) >= top_k:
            break
    return results


# ---------------------------------------------------------------------------
# Main experiment
# ---------------------------------------------------------------------------

def run_experiment(n: int = 13, schema: str = "tenant_internal") -> None:
    print(f"\n{'='*70}")
    print(f"ModuleDesk Embedding A/B Experiment")
    print(f"Schema: {schema} | Eval tickets: n={n}")
    print(f"{'='*70}\n")

    from supporthub.app.config import config
    from supporthub.app.db import session_scope, set_worker_tenant_schema

    if not config.AI_API_KEY:
        print("ERROR: AI_API_KEY not set in env.")
        sys.exit(1)

    set_worker_tenant_schema(schema)

    import openai
    client = openai.OpenAI(api_key=config.AI_API_KEY, base_url=config.AI_BASE_URL)
    costs = Costs()

    # --- Strategy parameters ---
    S0_MODEL = "text-embedding-3-small"
    S0_DIMS  = 512

    S1_MODEL = "text-embedding-3-small"
    S1_DIMS  = 512

    # S2: use text-embedding-3-small @ 1536 (full dims, no MRL truncation).
    # text-embedding-3-large @ 3072 would be ~6.5x more expensive per token
    # and adds significant runtime; 3-small @ 1536 tests the "more dims" axis
    # independently of the model-scale axis. If 3-small @ 1536 is flat vs S1,
    # we'll note that 3-large is the remaining untested lever.
    S2_MODEL = "text-embedding-3-small"
    S2_DIMS  = 1536

    # --- Select eval tickets ---
    from scripts.eval_draft_quality import select_eval_tickets
    print(f"Selecting eval tickets from {schema}...")
    with session_scope(schema=schema) as session:
        tickets = select_eval_tickets(session, n=n)
        session.expunge_all()
    print(f"  Found {len(tickets)} eligible tickets\n")
    if not tickets:
        print("ERROR: No eligible tickets found.")
        sys.exit(1)

    # --- Build corpus ---
    # Resolved threads = has at least one inbound + at least one outbound reply + has embedding.
    # Use JOIN-based query rather than correlated subqueries to avoid O(N) query cost.
    print("Building corpus from DB (resolved threads with embeddings)...")
    with session_scope(schema=schema) as session:
        from sqlalchemy import text as sa_text
        # Step 1: get all thread_ids that qualify (has embedding, inbound, and outbound)
        qualifying_tids = [row[0] for row in session.execute(sa_text("""
            SELECT DISTINCT te.thread_id
            FROM thread_embeddings te
            JOIN addons_messages m_out ON m_out.thread_id = te.thread_id
                AND m_out.direction = 'outbound'
            JOIN addons_messages m_in ON m_in.thread_id = te.thread_id
                AND m_in.direction = 'inbound'
            WHERE te.embedding_blob IS NOT NULL
              AND te.summary_text IS NOT NULL
        """)).fetchall()]
        print(f"  Qualifying thread_ids: {len(qualifying_tids)}")

        # Step 2: load summary_text from thread_embeddings
        emb_summary = {row[0]: (row[1], row[2]) for row in session.execute(sa_text("""
            SELECT thread_id, summary_text, product_id
            FROM thread_embeddings
            WHERE thread_id = ANY(:ids) AND summary_text IS NOT NULL
        """), {"ids": qualifying_tids}).fetchall()}

        # Step 3: load FIRST inbound per thread using DISTINCT ON
        inbound_map = {row[0]: row[1] for row in session.execute(sa_text("""
            SELECT DISTINCT ON (thread_id) thread_id, body_text
            FROM addons_messages
            WHERE thread_id = ANY(:ids) AND direction = 'inbound'
            ORDER BY thread_id, created_at ASC
        """), {"ids": qualifying_tids}).fetchall()}

        # Step 4: load LONGEST outbound per thread using DISTINCT ON
        outbound_map = {row[0]: row[1] for row in session.execute(sa_text("""
            SELECT DISTINCT ON (thread_id) thread_id, body_text
            FROM addons_messages
            WHERE thread_id = ANY(:ids) AND direction = 'outbound'
            ORDER BY thread_id, length(body_text) DESC
        """), {"ids": qualifying_tids}).fetchall()}

    # Build corpus dict from the 4 maps
    corpus_rows = []
    for tid in qualifying_tids:
        if tid in emb_summary and tid in inbound_map and tid in outbound_map:
            summary, product_id = emb_summary[tid]
            corpus_rows.append(type('Row', (), {
                'thread_id': tid,
                'summary_text': summary,
                'id_product': product_id,
                'first_inbound': inbound_map[tid],
                'longest_outbound': outbound_map[tid],
            })())

    # Also fetch the current production embeddings from DB for S0 corpus
    print("Loading production embeddings from DB for S0 corpus...")
    from supporthub.app.services.embedding_service import (
        embedding_index,
        EmbeddingService,
        unpack_embedding,
    )
    from supporthub.app.models import ThreadEmbeddings as TEModel
    with session_scope(schema=schema) as session:
        emb_rows = session.query(
            TEModel.thread_id,
            TEModel.embedding_blob,
            TEModel.dimensions,
        ).filter(TEModel.embedding_blob.isnot(None)).all()
        prod_embeddings: Dict[int, np.ndarray] = {}
        for tid, blob, dims in emb_rows:
            try:
                vec = unpack_embedding(blob, dims)
                prod_embeddings[tid] = vec
            except Exception:
                pass
    print(f"  Loaded {len(prod_embeddings)} production embeddings\n")

    # Build corpus dict
    corpus: Dict[int, Dict] = {}
    for row in corpus_rows:
        if row.first_inbound and row.longest_outbound:
            corpus[row.thread_id] = {
                "thread_id": row.thread_id,
                "summary_text": row.summary_text or "",
                "id_product": row.id_product,
                "first_inbound": row.first_inbound or "",
                "longest_outbound": row.longest_outbound or "",
            }

    # Only keep corpus entries that also have production embeddings (for S0)
    corpus_tids = [tid for tid in corpus.keys() if tid in prod_embeddings]
    print(f"  Corpus: {len(corpus_tids)} threads (have embedding + both inbound & outbound)\n")

    if len(corpus_tids) < 10:
        print(f"WARNING: Very small corpus ({len(corpus_tids)}). Results may not be meaningful.")

    # --- S0: Production baseline matrix (from stored embeddings, summary_text) ---
    print(f"[S0] Building production baseline matrix ({S0_MODEL} @ {S0_DIMS} dims on summary_text)...")
    s0_vecs = np.array([prod_embeddings[tid] for tid in corpus_tids], dtype=np.float32)
    # Normalize
    norms = np.linalg.norm(s0_vecs, axis=1, keepdims=True) + 1e-10
    s0_matrix_normed = s0_vecs / norms
    print(f"  S0 corpus matrix: {s0_matrix_normed.shape}")

    # --- Select candidate subset for S1/S2 re-embedding ---
    # Embedding all 6,000+ corpus threads would be expensive (~€0.06) and slow.
    # Instead we take the top-50 S0 neighbors per eval ticket (from the full production
    # index) plus a 300-thread random background sample. This covers the threads most
    # likely to be relevant, while keeping cost and runtime manageable.
    #
    # Design rationale: if S1/S2 embeddings would find a NEW relevant thread that S0
    # missed, it almost always lies within the top-100 S0 cosine neighbors of the
    # query — because the summary embedding captures enough signal to rank truly
    # relevant threads near the top, even if it can't distinguish the #1 from #3.
    # The random background sample (~300) guards against the edge case where the
    # relevant thread has very low S0 similarity (< 0.35) — which the RAG audit
    # already showed is rare (brute-force top-5 scored ≤0.52 for no-precedent tickets).

    # Load production embedding index for full-corpus S0 scan
    print("\nLoading production embedding index for full-corpus neighbor scan...")
    embedding_index.load_from_db(schema=schema)
    print(f"  Index loaded: {embedding_index.count} vectors")

    CANDIDATE_PER_TICKET = 50   # top S0 neighbors per ticket
    RANDOM_BACKGROUND = 300     # random corpus threads as background

    candidate_tids: set = set()
    # Cache: ticket_id -> (inbound_text, s0_query_vec)
    ticket_query_cache: Dict[int, tuple] = {}

    # Pre-load the full index into local arrays for the scan (avoid repeated lock acquisitions)
    with embedding_index._lock:
        idx_tids_scan = list(embedding_index._thread_ids)
        idx_vecs_scan = embedding_index._vectors.copy()
    idx_norms = np.linalg.norm(idx_vecs_scan, axis=1, keepdims=True) + 1e-10
    idx_vecs_normed = idx_vecs_scan / idx_norms

    for item in tickets:
        # Load inbound and embed with S0
        with session_scope(schema=schema) as session:
            from sqlalchemy import text as sa_text3
            row = session.execute(sa_text3(
                "SELECT body_text FROM addons_messages WHERE id = :id"
            ), {"id": item["inbound_msg_id"]}).one_or_none()
        if not row or not row.body_text:
            continue
        inbound = row.body_text[:8000]
        try:
            resp = client.embeddings.create(input=inbound, model=S0_MODEL, dimensions=S0_DIMS)
            if resp.usage:
                costs.record("s0_scan", S0_MODEL, resp.usage.prompt_tokens)
            qvec = np.array(resp.data[0].embedding, dtype=np.float32)
        except Exception as exc:
            print(f"  WARN: Could not embed ticket {item['ticket_id']}: {exc}")
            continue
        # Cache for reuse in per-ticket loop (avoid re-embedding)
        ticket_query_cache[item["ticket_id"]] = (row.body_text, qvec)
        # Get top-N neighbors from full index (no floor, just raw cosine)
        qnorm = qvec / (np.linalg.norm(qvec) + 1e-10)
        sims = idx_vecs_normed @ qnorm
        top_indices = np.argsort(sims)[::-1][:CANDIDATE_PER_TICKET]
        for idx in top_indices:
            tid = idx_tids_scan[idx]
            if tid in corpus and tid != item["thread_id"]:
                candidate_tids.add(tid)

    # Add random background
    import random as _random
    _random.seed(42)
    background = [t for t in corpus_tids if t not in candidate_tids]
    _random.shuffle(background)
    candidate_tids.update(background[:RANDOM_BACKGROUND])
    # Ensure eval tickets' own threads are excluded from corpus
    eval_thread_ids = {t["thread_id"] for t in tickets}
    candidate_tids -= eval_thread_ids

    # Build ordered candidate list; intersect with corpus
    candidate_tids_list = [t for t in corpus_tids if t in candidate_tids]
    print(f"\nCandidate subset for S1/S2: {len(candidate_tids_list)} threads "
          f"(top-{CANDIDATE_PER_TICKET} neighbors per ticket + {RANDOM_BACKGROUND} random background)")

    # --- S1: Full-thread-text corpus, same model/dims ---
    print(f"\n[S1] Embedding {len(candidate_tids_list)} threads on full text ({S1_MODEL} @ {S1_DIMS} dims)...")
    s1_corpus_texts = []
    for tid in candidate_tids_list:
        c = corpus[tid]
        full = (
            f"Customer question:\n{c['first_inbound'][:2000]}\n\n"
            f"Support reply:\n{c['longest_outbound'][:1500]}"
        )
        s1_corpus_texts.append(full)

    s1_corpus_vecs = embed_texts(client, s1_corpus_texts, S1_MODEL, S1_DIMS, costs, "s1_corpus")
    norms = np.linalg.norm(s1_corpus_vecs, axis=1, keepdims=True) + 1e-10
    s1_matrix_normed = s1_corpus_vecs / norms
    print(f"  S1 corpus matrix: {s1_matrix_normed.shape}")

    # --- S2: Full-thread-text corpus, larger dims ---
    print(f"\n[S2] Embedding {len(candidate_tids_list)} threads on full text ({S2_MODEL} @ {S2_DIMS} dims)...")
    s2_corpus_vecs = embed_texts(client, s1_corpus_texts, S2_MODEL, S2_DIMS, costs, "s2_corpus")
    norms = np.linalg.norm(s2_corpus_vecs, axis=1, keepdims=True) + 1e-10
    s2_matrix_normed = s2_corpus_vecs / norms
    print(f"  S2 corpus matrix: {s2_matrix_normed.shape}")

    embedding_cost_so_far = costs.total()
    print(f"\nCorpus embedding cost: €{embedding_cost_so_far:.5f}")

    # ---------------------------------------------------------------------------
    # Per-ticket retrieval + judge
    # ---------------------------------------------------------------------------
    TOP_K = 3

    ticket_results = []
    eval_tids = {t["thread_id"] for t in tickets}

    print(f"\n{'='*70}")
    print(f"Running per-ticket retrieval + relevance judge (TOP_K={TOP_K})...")
    print(f"{'='*70}\n")

    for i, item in enumerate(tickets, 1):
        tid_self = item["thread_id"]
        exclude = {tid_self}

        print(f"[{i}/{len(tickets)}] ticket={item['ticket_id']} thread={tid_self} | {item['product_name']}")

        # Load inbound text from cache (already loaded in the candidate scan phase)
        if item["ticket_id"] not in ticket_query_cache:
            print("  SKIP: not in query cache (embedding failed earlier)")
            continue
        inbound_text, s0_query_vec = ticket_query_cache[item["ticket_id"]]
        if not inbound_text.strip():
            print("  SKIP: no inbound body")
            continue

        # S0 top-k (against full corpus — uses stored embeddings, no new API call)
        s0_top = cosine_top_k(s0_query_vec, s0_matrix_normed, corpus_tids, exclude, TOP_K)

        # --- S1: embed query on inbound question, same model/dims ---
        # Note: S1 and S2 search the candidate subset (top-50 neighbors + random background)
        # rather than the full 6k corpus. See corpus-building comment above for rationale.
        try:
            resp = client.embeddings.create(
                input=inbound_text[:8000],
                model=S1_MODEL,
                dimensions=S1_DIMS,
            )
            if resp.usage:
                costs.record("s1_query", S1_MODEL, resp.usage.prompt_tokens)
            s1_query_vec = np.array(resp.data[0].embedding, dtype=np.float32)
        except Exception as exc:
            print(f"  ERROR S1 embed: {exc}")
            continue

        s1_top = cosine_top_k(s1_query_vec, s1_matrix_normed, candidate_tids_list, exclude, TOP_K)

        # --- S2: embed query on inbound question, larger dims ---
        try:
            resp = client.embeddings.create(
                input=inbound_text[:8000],
                model=S2_MODEL,
                dimensions=S2_DIMS,
            )
            if resp.usage:
                costs.record("s2_query", S2_MODEL, resp.usage.prompt_tokens)
            s2_query_vec = np.array(resp.data[0].embedding, dtype=np.float32)
        except Exception as exc:
            print(f"  ERROR S2 embed: {exc}")
            continue

        s2_top = cosine_top_k(s2_query_vec, s2_matrix_normed, candidate_tids_list, exclude, TOP_K)

        # --- Judge top-3 for each strategy ---
        def judge_top(top_k_results: List[Tuple[int, float]], strategy_label: str) -> Tuple[bool, Optional[Dict]]:
            """Judge top-k retrieved; return (has_relevant, best_relevant_dict)."""
            has_relevant = False
            best = None
            for rank, (ret_tid, sim) in enumerate(top_k_results, 1):
                if ret_tid not in corpus:
                    continue
                # Only judge if similarity is at least plausible (> 0.35) to save cost
                if sim < 0.35:
                    continue
                precedent_q = corpus[ret_tid]["first_inbound"]
                verdict = judge_relevance(inbound_text, precedent_q, client, costs)
                print(f"    [{strategy_label}] rank={rank} tid={ret_tid} sim={sim:.3f} "
                      f"relevant={verdict['relevant']} conf={verdict['confidence']} | {verdict['reason'][:60]}")
                if verdict["relevant"] and not has_relevant:
                    has_relevant = True
                    best = {"thread_id": ret_tid, "sim": sim, "rank": rank, **verdict}
            return has_relevant, best

        print(f"  S0 top-{TOP_K}: {[(tid, round(sim,3)) for tid,sim in s0_top]}")
        s0_has, s0_best = judge_top(s0_top, "S0")

        print(f"  S1 top-{TOP_K}: {[(tid, round(sim,3)) for tid,sim in s1_top]}")
        s1_has, s1_best = judge_top(s1_top, "S1")

        print(f"  S2 top-{TOP_K}: {[(tid, round(sim,3)) for tid,sim in s2_top]}")
        s2_has, s2_best = judge_top(s2_top, "S2")

        s0_label = "has-precedent" if s0_has else "no-precedent"
        s1_label = "has-precedent" if s1_has else "no-precedent"
        s2_label = "has-precedent" if s2_has else "no-precedent"

        newly_found_s1 = s1_has and not s0_has
        newly_found_s2 = s2_has and not s0_has

        print(f"  => S0={s0_label} | S1={s1_label} (new={newly_found_s1}) | S2={s2_label} (new={newly_found_s2})")
        print()

        ticket_results.append({
            "ticket_id": item["ticket_id"],
            "thread_id": tid_self,
            "product_name": item["product_name"],
            "inbound_preview": inbound_text[:200],
            "s0_has_precedent": s0_has,
            "s1_has_precedent": s1_has,
            "s2_has_precedent": s2_has,
            "s1_newly_found": newly_found_s1,
            "s2_newly_found": newly_found_s2,
            "s0_top": [(tid, round(sim, 4)) for tid, sim in s0_top],
            "s1_top": [(tid, round(sim, 4)) for tid, sim in s1_top],
            "s2_top": [(tid, round(sim, 4)) for tid, sim in s2_top],
            "s0_best": s0_best,
            "s1_best": s1_best,
            "s2_best": s2_best,
        })

    # ---------------------------------------------------------------------------
    # Aggregate recall
    # ---------------------------------------------------------------------------
    N = len(ticket_results)
    s0_recall = sum(1 for r in ticket_results if r["s0_has_precedent"])
    s1_recall = sum(1 for r in ticket_results if r["s1_has_precedent"])
    s2_recall = sum(1 for r in ticket_results if r["s2_has_precedent"])
    s1_newly = sum(1 for r in ticket_results if r["s1_newly_found"])
    s2_newly = sum(1 for r in ticket_results if r["s2_newly_found"])

    total_cost = costs.total()
    run_date = time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())

    print(f"\n{'='*70}")
    print(f"EMBEDDING A/B RESULTS — {run_date}")
    print(f"{'='*70}")
    print(f"Eval tickets: {N}")
    print(f"Corpus size:  {len(corpus_tids)} threads\n")

    # Recall table
    print(f"{'Strategy':<40} {'Recall@3':<12} {'Has-prec':<10} {'Newly found'}")
    print(f"{'-'*70}")
    print(f"{'S0: 3-small@512 / summary_text (prod)':<40} {s0_recall}/{N}         {s0_recall:<10} —")
    print(f"{'S1: 3-small@512 / full thread text':<40} {s1_recall}/{N}         {s1_recall:<10} +{s1_newly} tickets")
    print(f"{'S2: 3-small@1536 / full thread text':<40} {s2_recall}/{N}         {s2_recall:<10} +{s2_newly} tickets")

    # Per-ticket breakdown
    print(f"\n{'Ticket':<10} {'Product':<35} S0       S1       S2")
    print(f"{'-'*75}")
    for r in ticket_results:
        s0_sym = "HAS-PREC" if r["s0_has_precedent"] else "no-prec "
        s1_sym = "HAS-PREC" if r["s1_has_precedent"] else "no-prec "
        s2_sym = "HAS-PREC" if r["s2_has_precedent"] else "no-prec "
        s1_new = " +NEW" if r["s1_newly_found"] else ""
        s2_new = " +NEW" if r["s2_newly_found"] else ""
        print(f"{r['ticket_id']:<10} {r['product_name'][:34]:<35} {s0_sym} {s1_sym}{s1_new:<5} {s2_sym}{s2_new}")

    # Cost
    print(f"\nTotal experiment cost: €{total_cost:.5f}")
    print("Cost breakdown:")
    for k, v in sorted(costs.breakdown().items()):
        print(f"  {k}: €{v:.5f}")

    # ---------------------------------------------------------------------------
    # Verdict
    # ---------------------------------------------------------------------------
    # "Material improvement" = at least +2 newly-found tickets OR recall gain >= 15pp
    MATERIAL_THRESHOLD = max(2, round(N * 0.15))

    print(f"\n{'='*70}")
    print("VERDICT")
    print(f"{'='*70}")

    s1_material = s1_newly >= MATERIAL_THRESHOLD
    s2_material = s2_newly >= MATERIAL_THRESHOLD
    any_material = s1_material or s2_material

    if any_material:
        print(f"RAG IS IMPROVABLE via better embeddings.")
        if s1_material:
            print(f"  S1 (full text, same model/dims) found +{s1_newly} new precedents vs S0.")
            print(f"  Recommendation: re-index corpus on full thread text (question + longest reply).")
            print(f"  Schema/dim change NOT required — stays at text-embedding-3-small@512.")
        if s2_material and not s1_material:
            print(f"  S2 (full text, 1536 dims) found +{s2_newly} new precedents vs S0.")
            print(f"  Recommendation: re-index on full text + raise EMBEDDING_DIMENSIONS to 1536.")
            print(f"  NOTE: this requires a schema migration (thread_embeddings.embedding_blob grows ~3x).")
        elif s2_material and s1_material:
            s2_extra = s2_newly - s1_newly
            if s2_extra >= 1:
                print(f"  S2 additionally found +{s2_extra} tickets beyond S1 — larger dims add marginal gain.")
            else:
                print(f"  S2 did NOT improve over S1 — extra dims add no benefit; stick with S1 re-index.")
        # Production re-index cost estimate
        print(f"\n  Re-index cost estimate ({len(corpus_tids)} corpus threads on full text @ 3-small/512):")
        avg_full_text_tokens = 600   # rough: 2000+1500 chars / 4 chars per token average
        est_tokens = len(corpus_tids) * avg_full_text_tokens
        est_cost = tokens_to_eur("text-embedding-3-small", est_tokens)
        print(f"  ~{est_tokens:,} tokens → €{est_cost:.4f} one-time (vs €{est_cost*3:.4f} for 1536-dims)")
    else:
        print(f"CONFIRMED: ceiling is genuine corpus sparsity, NOT embedding quality.")
        print(f"  S1 found +{s1_newly} new precedents (threshold for material = {MATERIAL_THRESHOLD}).")
        print(f"  S2 found +{s2_newly} new precedents.")
        print(f"  Better text / bigger dims do NOT materially improve recall.")
        print(f"  Verdict: do NOT re-index. The no-precedent tickets lack a genuine match in the corpus.")
        print(f"  Recommended lever: corpus growth (more resolved tickets) or per-product knowledge base.")

    print(f"\nExperiment cost: €{total_cost:.5f}")

    # ---------------------------------------------------------------------------
    # Save JSON results
    # ---------------------------------------------------------------------------
    out_dir = Path(__file__).resolve().parents[1] / "tmp"
    out_dir.mkdir(exist_ok=True)
    json_path = out_dir / "eval-embedding-ab.json"
    result_data = {
        "run_date": run_date,
        "schema": schema,
        "n_tickets": N,
        "corpus_size": len(corpus_tids),
        "strategies": {
            "S0": {"model": S0_MODEL, "dims": S0_DIMS, "corpus_text": "summary_text"},
            "S1": {"model": S1_MODEL, "dims": S1_DIMS, "corpus_text": "full_thread_text"},
            "S2": {"model": S2_MODEL, "dims": S2_DIMS, "corpus_text": "full_thread_text"},
        },
        "recall_at_3": {
            "S0": s0_recall,
            "S1": s1_recall,
            "S2": s2_recall,
        },
        "newly_found": {
            "S1": s1_newly,
            "S2": s2_newly,
        },
        "material_threshold": MATERIAL_THRESHOLD,
        "verdict": "RAG_IMPROVABLE" if any_material else "CORPUS_SPARSITY_CONFIRMED",
        "total_cost_eur": round(total_cost, 6),
        "per_ticket": ticket_results,
    }
    with open(json_path, "w", encoding="utf-8") as f:
        json.dump(result_data, f, indent=2, ensure_ascii=False)
    print(f"\nResults JSON → {json_path}")

    # ---------------------------------------------------------------------------
    # Update tmp/eval-draft-progress.md
    # ---------------------------------------------------------------------------
    progress_path = Path(__file__).resolve().parents[1] / "tmp" / "eval-draft-progress.md"
    verdict_short = (
        "RAG IS IMPROVABLE" if any_material else
        "CORPUS SPARSITY CONFIRMED — embedding quality is NOT the ceiling"
    )
    s1_detail = f"+{s1_newly} newly-found" if s1_material else f"flat (+{s1_newly})"
    s2_detail = f"+{s2_newly} newly-found" if s2_material else f"flat (+{s2_newly})"

    entry = f"""
---

## Embedding A/B Experiment — {run_date}

**Script:** `scripts/eval_embedding_ab.py`
**Branch:** `overnight/2026-07-02-ai-drafts`
**Eval tickets:** {N} | **Corpus:** {len(corpus_tids)} threads

### Recall@3 Table

| Strategy | Model | Dims | Corpus text | Recall@3 | Newly-found vs S0 |
|----------|-------|------|-------------|----------|------------------|
| S0 (prod baseline) | text-embedding-3-small | 512 | summary_text (1-2 sent GPT summary) | {s0_recall}/{N} | — |
| S1 (full text, same model) | text-embedding-3-small | 512 | full thread text (question + longest reply) | {s1_recall}/{N} | {s1_detail} |
| S2 (full text, 1536 dims) | text-embedding-3-small | 1536 | full thread text (question + longest reply) | {s2_recall}/{N} | {s2_detail} |

### Verdict

**{verdict_short}**

{"S1/S2 materially raise recall (≥2 newly-found). Recommend production re-index on full thread text." if any_material else f"S1/S2 do NOT beat S0 by the material threshold (≥{MATERIAL_THRESHOLD} newly-found). The no-precedent ceiling is genuine corpus sparsity, not embedding quality. No re-index warranted."}

Experiment cost: **€{total_cost:.5f}**
"""

    with open(progress_path, "a", encoding="utf-8") as f:
        f.write(entry)
    print(f"Progress log updated → {progress_path}")

    return result_data


# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Embedding A/B Experiment for ModuleDesk")
    parser.add_argument("--n", type=int, default=13, help="Number of eval tickets (default: 13)")
    parser.add_argument("--schema", default="tenant_internal", help="Tenant schema (default: tenant_internal)")
    args = parser.parse_args()

    run_experiment(n=args.n, schema=args.schema)