cleveragents-core/tools/llm_activity_scraper.py

"""OpenCode session-archive scraper for the ``llm_activity`` table.

Reads the JSON archives that ``tools/_opencode_worker.run_session_blocking``
writes to ``.dispatcher-logs/sessions/*.json`` after every terminal
worker session (completed / timeout / transport-error) and emits one
``llm_activity`` row per assistant turn into the SQLite cache.

Why a scraper and not per-worker writers
----------------------------------------
The worker already does the hard part: it walks the subagent tree via
OpenCode's ``parentID`` chain (see ``_archive_subagent_tree``) and
writes each subagent as its own archive file. By feeding off the
archives, this scraper captures every LLM turn the OpenCode runtime
performed — top-level wrappers, ``task``-tool subagents at any depth,
even retried sub-sessions — without requiring any worker to remember
to instrument itself. The one gap this leaves is sessions whose
wrapper process is SIGKILL'd before the archive write completes;
a live-OpenCode poller (TODO) would close that gap.

What ends up in each row
------------------------
Per-turn fields are extracted from the raw OpenCode message at
``info`` level — the same shape ``_summarize_per_turn`` consumes:

    info.id            → message_id   (UNIQUE — dedup key for re-runs)
    info.sessionID     → session_id
    info.modelID       → model
    info.providerID    → provider
    info.tokens.input  → tokens_in  (fresh prompt tokens — non-cached)
    info.tokens.output → tokens_out (+ reasoning_tokens; see below)
    info.tokens.cache.read  → cached_tokens
    info.time.created  → started_at (ISO 8601)
    info.time.completed→ ended_at   (NULL for in-flight turns)

Reasoning tokens (Anthropic thinking / OpenAI reasoning) are folded
into ``tokens_out`` because every provider we use bills them at the
output rate — keeping them separate would underestimate cost without
schema-level price-table changes. The raw per-turn breakdown (incl.
the OpenCode-reported ``cost`` field and the original cache write/read
split) is preserved into the ``raw`` JSON column so a richer cost
calculator can re-derive without re-scraping.

Subagent attribution: ``parent_session_id`` and ``subagent_depth`` come
from the archive header. A top-level wrapper gets NULL/None for both;
a depth-N subagent gets the wrapper's session id and its own depth so
the dashboard can roll cost up or down the call tree.

Idempotency
-----------
The v6 schema added a partial UNIQUE INDEX on
``llm_activity(message_id) WHERE message_id IS NOT NULL``. Every row
this scraper writes carries ``message_id`` from the OpenCode assistant
message, so ``INSERT OR IGNORE`` collapses re-scrapes to no-ops. This
means it's safe to:

- run on every dispatcher cycle (cheap)
- run from cron at any cadence
- back-fill historical archives without worrying about duplicates

CLI
---
    python3 tools/llm_activity_scraper.py [--archive-dir PATH]
                                          [--since-hours N]
                                          [--db PATH]
                                          [--verbose]

Without flags, scrapes every archive in ``.dispatcher-logs/sessions/``
into the default cache DB ``tools/.cache/forgejo.sqlite`` (respecting
the ``FORGEJO_OWNER``/``FORGEJO_REPO`` partitioning).
"""
from __future__ import annotations

import argparse
import hashlib
import importlib.util
import json
import logging
import os
import re
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any

logger = logging.getLogger("llm_activity_scraper")

REPO_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_ARCHIVE_DIR = REPO_ROOT / ".dispatcher-logs" / "sessions"

# Tag → PR number. Matches the wrapper tags emitted by dispatch_review
# (``AUTO-REV-PR-30``) and dispatch_implementer (``AUTO-IMP-PR-30``),
# plus the future ``AUTO-MIN-PR-N`` / ``AUTO-CON-PR-N`` shapes. Tags
# without a PR component (e.g. dispatcher-overhead sessions) parse to
# None so the row carries NULL.
_TAG_PR_RE = re.compile(r"AUTO-[A-Z]+-PR-(\d+)")


def _load_cache_module():
    """Import ``tools/_pipeline_cache.py`` as a top-level module by file
    path so this script runs without the package being on sys.path.

    Mirrors the loader pattern used by other ``tools/*.py`` scripts.
    """
    path = Path(__file__).resolve().parent / "_pipeline_cache.py"
    spec = importlib.util.spec_from_file_location("_pipeline_cache", path)
    assert spec and spec.loader
    mod = importlib.util.module_from_spec(spec)
    sys.modules["_pipeline_cache"] = mod
    spec.loader.exec_module(mod)
    return mod


def _ms_to_iso(ms: Any) -> str | None:
    """OpenCode emits timestamps as integer milliseconds since epoch.
    Convert to ISO-8601 UTC; return None for missing / non-numeric."""
    if not isinstance(ms, (int, float)):
        return None
    try:
        return datetime.fromtimestamp(ms / 1000.0, tz=timezone.utc).isoformat()
    except (ValueError, OSError, OverflowError):
        return None


def _pr_number_from_tag(tag: str | None) -> int | None:
    if not isinstance(tag, str):
        return None
    m = _TAG_PR_RE.search(tag)
    if not m:
        return None
    try:
        return int(m.group(1))
    except ValueError:
        return None


def _first_user_prompt_hash(messages: list[Any]) -> str | None:
    """SHA-256 the concatenated text parts of the first user message.

    The session's first user message IS the prompt that kicked off
    the LLM call chain — same first prompt → same dispatch decision.
    We hash it (not store it) so the column stays small and we can
    do ``GROUP BY prompt_hash`` without risking PII / secret leakage
    into the cache DB.

    Returns ``None`` when the archive has no user message with text
    parts (defensive; should be rare).
    """
    for m in messages:
        if not isinstance(m, dict):
            continue
        info = m.get("info") if isinstance(m.get("info"), dict) else m
        if not isinstance(info, dict) or info.get("role") != "user":
            continue
        parts = m.get("parts") or []
        chunks = [
            p.get("text", "")
            for p in parts
            if isinstance(p, dict) and p.get("type") == "text"
            and isinstance(p.get("text"), str)
        ]
        if not chunks:
            return None
        return hashlib.sha256("".join(chunks).encode("utf-8")).hexdigest()
    return None


def _turn_rows_from_archive(archive: dict[str, Any]) -> list[dict[str, Any]]:
    """Extract one ``llm_activity``-shaped dict per assistant turn from
    one parsed archive payload.

    Skips:
    - non-assistant messages (user / system turns have no token usage)
    - assistant messages with no ``modelID`` (defensive — would violate
      the NOT NULL constraint on ``model`` and is unexpected; logged at
      DEBUG so a real malformed archive surfaces in --verbose runs)

    Returns an empty list rather than raising on a malformed archive so
    a single corrupt file doesn't take down a directory-wide scrape.
    """
    session_id = archive.get("session_id")
    parent_session_id = archive.get("parent_session_id")
    # Normalize top-level archives to depth 0 so dashboards can filter
    # "subagents only" with ``WHERE subagent_depth > 0`` instead of
    # juggling NULLs. The archive writer leaves the field None on
    # top-level wrappers (see _opencode_worker._archive_payload).
    raw_depth = archive.get("subagent_depth")
    subagent_depth = (
        int(raw_depth) if isinstance(raw_depth, (int, float)) else 0
    )
    agent = archive.get("agent") or "unknown"
    # Hash the session's first user prompt once — applied to every
    # assistant turn so ``GROUP BY prompt_hash`` reveals cycles that
    # dispatched the same worker with the same input.
    prompt_hash = _first_user_prompt_hash(archive.get("messages") or [])
    tag = archive.get("tag")
    outcome = archive.get("status")
    pr_number = _pr_number_from_tag(tag)

    messages = archive.get("messages") or []
    if not isinstance(messages, list):
        return []

    rows: list[dict[str, Any]] = []
    for m in messages:
        if not isinstance(m, dict):
            continue
        info = m.get("info") if isinstance(m.get("info"), dict) else m
        if not isinstance(info, dict):
            continue
        if info.get("role") != "assistant":
            continue

        model = info.get("modelID")
        if not isinstance(model, str) or not model:
            logger.debug(
                "skipping assistant turn with no modelID "
                "(session=%s, message=%s)",
                session_id, info.get("id"),
            )
            continue

        tokens = info.get("tokens") if isinstance(info.get("tokens"), dict) else {}
        time_info = info.get("time") if isinstance(info.get("time"), dict) else {}
        cache_info = tokens.get("cache") if isinstance(tokens.get("cache"), dict) else {}

        input_tok = tokens.get("input")
        output_tok = tokens.get("output")
        reasoning_tok = tokens.get("reasoning")
        cache_read = cache_info.get("read")
        cache_write = cache_info.get("write")

        # Roll reasoning into tokens_out (billed at output rate by every
        # provider we use); cache_write into tokens_in (billed alongside
        # fresh prompt tokens, sometimes at a premium — preserved in
        # ``raw`` for future cost-calc refinements).
        tokens_in = (int(input_tok) if isinstance(input_tok, (int, float)) else 0) \
            + (int(cache_write) if isinstance(cache_write, (int, float)) else 0)
        tokens_out = (int(output_tok) if isinstance(output_tok, (int, float)) else 0) \
            + (int(reasoning_tok) if isinstance(reasoning_tok, (int, float)) else 0)
        cached = int(cache_read) if isinstance(cache_read, (int, float)) else 0

        started_at = _ms_to_iso(time_info.get("created"))
        ended_at = _ms_to_iso(time_info.get("completed"))
        if not started_at:
            # Without a start time we can't honour the NOT NULL
            # constraint on started_at. Fall back to the archive's
            # session-level started_at so the row still lands; better
            # to have a slightly imprecise timestamp than to drop a
            # cost-bearing turn entirely.
            started_at = archive.get("started_at")
        if not started_at:
            logger.debug(
                "skipping assistant turn with no recoverable started_at "
                "(session=%s, message=%s)",
                session_id, info.get("id"),
            )
            continue

        rows.append(
            {
                "started_at": started_at,
                "ended_at": ended_at,
                "agent": agent,
                "pr_number": pr_number,
                "session_tag": tag,
                "model": model,
                "tokens_in": tokens_in,
                "tokens_out": tokens_out,
                "cached_tokens": cached,
                "outcome": outcome,
                "session_id": session_id,
                "message_id": info.get("id"),
                "provider": info.get("providerID"),
                "parent_session_id": parent_session_id,
                "subagent_depth": subagent_depth,
                "prompt_hash": prompt_hash,
                # Forensic detail kept out of the indexed columns:
                "raw_input_tokens": input_tok,
                "raw_output_tokens": output_tok,
                "raw_reasoning_tokens": reasoning_tok,
                "raw_cache_read": cache_read,
                "raw_cache_write": cache_write,
                "raw_opencode_cost": info.get("cost"),
                "raw_finish_reason": (info.get("finish") or {}).get("reason")
                    if isinstance(info.get("finish"), dict) else None,
            }
        )
    return rows


def scrape_archive(archive_path: Path, cache: Any) -> tuple[int, int]:
    """Scrape one archive file. Returns ``(inserted, duplicate)``.

    Duplicates (rows whose ``message_id`` is already in the table) are
    counted but not re-inserted, courtesy of the partial unique index
    and ``INSERT OR IGNORE`` in :meth:`PipelineCache.upsert_llm_activity_batch`.
    The batch path commits once per archive rather than once per
    turn — the backfill on 435 archives went from ~3.6k fsyncs to ~435.
    """
    try:
        with archive_path.open("r", encoding="utf-8") as f:
            archive = json.load(f)
    except (OSError, json.JSONDecodeError) as e:
        logger.warning("skipping unreadable archive %s: %s", archive_path.name, e)
        return 0, 0
    if not isinstance(archive, dict):
        logger.warning("skipping non-object archive %s", archive_path.name)
        return 0, 0

    rows = _turn_rows_from_archive(archive)
    if not rows:
        return 0, 0
    result = cache.upsert_llm_activity_batch(rows)
    return result["inserted"], result["duplicate"]


def scrape_directory(
    archive_dir: Path,
    cache: Any,
    since: datetime | None = None,
) -> dict[str, int]:
    """Scrape every ``*.json`` archive in ``archive_dir``.

    ``since`` filters by file mtime — useful for periodic runs that
    want to avoid re-stat'ing the entire history every cycle, though
    dedup makes a full re-scrape safe (just wasteful at scale).
    """
    if not archive_dir.is_dir():
        logger.info("archive dir does not exist: %s", archive_dir)
        return {
            "archives_scanned": 0,
            "archives_skipped_old": 0,
            "turns_inserted": 0,
            "turns_duplicate": 0,
        }

    scanned = skipped_old = inserted_total = duplicate_total = 0
    since_ts = since.timestamp() if since else None
    for path in sorted(archive_dir.glob("*.json")):
        if since_ts is not None:
            try:
                if path.stat().st_mtime < since_ts:
                    skipped_old += 1
                    continue
            except OSError:
                continue
        scanned += 1
        ins, dup = scrape_archive(path, cache)
        inserted_total += ins
        duplicate_total += dup
    return {
        "archives_scanned": scanned,
        "archives_skipped_old": skipped_old,
        "turns_inserted": inserted_total,
        "turns_duplicate": duplicate_total,
    }


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description="Ingest OpenCode session archives into the "
                    "llm_activity table.",
    )
    parser.add_argument(
        "--archive-dir",
        type=Path,
        default=DEFAULT_ARCHIVE_DIR,
        help="Directory of session archives "
             "(default: .dispatcher-logs/sessions/)",
    )
    parser.add_argument(
        "--since-hours",
        type=float,
        default=None,
        help="Only scan archives modified in the last N hours "
             "(default: all archives; dedup makes this safe).",
    )
    parser.add_argument(
        "--db",
        type=Path,
        default=None,
        help="SQLite cache path "
             "(default: tools/.cache/forgejo.sqlite — see _pipeline_cache.py)",
    )
    parser.add_argument(
        "--verbose", "-v",
        action="store_true",
        help="Enable DEBUG logging (e.g. malformed-turn diagnostics).",
    )
    args = parser.parse_args(argv)

    logging.basicConfig(
        level=logging.DEBUG if args.verbose else logging.INFO,
        format="%(levelname)s %(name)s: %(message)s",
    )

    cache_mod = _load_cache_module()
    cache = cache_mod.PipelineCache(args.db) if args.db else cache_mod.PipelineCache.open()

    since = None
    if args.since_hours is not None:
        since = datetime.now(timezone.utc) - timedelta(hours=args.since_hours)

    summary = scrape_directory(args.archive_dir, cache, since=since)
    logger.info(
        "scrape complete: %d archives scanned (%d skipped as old), "
        "%d turns inserted, %d duplicates ignored",
        summary["archives_scanned"],
        summary["archives_skipped_old"],
        summary["turns_inserted"],
        summary["turns_duplicate"],
    )
    # Emit a one-line machine-readable summary too so a wrapper script
    # / cron job can parse the result without re-running.
    print(json.dumps(summary))
    return 0


if __name__ == "__main__":
    sys.exit(main())