drew
88b9373fa9
Closes the cost-tracking instrumentation gap: the telemetry console's Cost tab read from an empty ``llm_activity`` table because nothing in production wrote to it. The scraper walks the OpenCode session archives that ``_opencode_worker`` already writes (including subagent trees via the BFS-walked ``parentID`` chain) and emits one row per assistant turn. Folded into the existing PR-State Warmer loop so it runs on the same 30s cadence without spinning a new sidecar. Schema (v6): - ``llm_activity`` grows ``session_id`` / ``message_id`` / ``provider`` / ``parent_session_id`` / ``subagent_depth`` columns - Partial UNIQUE INDEX on ``message_id`` makes re-scrapes idempotent - v5→v6 migration ALTER-gated on column existence (safe to re-run) Scraper (``tools/llm_activity_scraper.py``): - Reads ``.dispatcher-logs/sessions/*.json``, one row per assistant turn - Folds reasoning tokens into ``tokens_out`` and cache-write into ``tokens_in`` (preserves raw breakdown in ``raw`` JSON for future cost-calc refinements) - Normalises ``subagent_depth=0`` at top level so dashboards can filter ``WHERE subagent_depth > 0`` cleanly - Batch INSERT OR IGNORE via new ``PipelineCache.upsert_llm_activity_batch`` — one fsync per archive, not per turn Warmer integration: - First tick: full backfill of the archive directory - Subsequent ticks: 1h lookback via ``since=`` filter - Scraper failures are logged and swallowed — PR-state job stays load-bearing and unaffected - ``LLM_ACTIVITY_SCRAPER_DISABLE=1`` env kill switch Renames (mechanical, atomic): - ``tools/_forgejo_cache.py`` → ``tools/_pipeline_cache.py`` - ``ForgejoCache`` class → ``PipelineCache`` - Both reflect the module's broader scope (Forgejo data + pipeline telemetry tables); on-disk filename ``forgejo.sqlite`` and ``FORGEJO_*`` env vars are kept for compatibility Verified end-to-end on real archives: 435 archives → 3595 turns ingested (2873 from subagents) across 8 models / 5 providers / 9 PRs. Re-runs insert 0, dedup 3595. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
119 lines
3.5 KiB
Python
Executable File
119 lines
3.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Sync the local Forgejo cache at ``tools/.cache/forgejo.sqlite``.
|
|
|
|
This tool is the human-facing wrapper around ``_pipeline_cache.PipelineCache``.
|
|
On a cold cache it pulls the full master commit history and every PR the
|
|
repo has ever had. On a warm cache it pulls only deltas (new commits since
|
|
last sync, PRs whose ``updated_at`` has advanced).
|
|
|
|
Usage
|
|
-----
|
|
|
|
Typical incremental refresh::
|
|
|
|
python3 tools/forgejo-sync.py
|
|
|
|
Seed a fresh cache from scratch (destructive)::
|
|
|
|
python3 tools/forgejo-sync.py --full
|
|
|
|
After any sync, back-fill ``merged_by`` details for every merged PR so
|
|
downstream reports don't have to fetch per-PR detail on read::
|
|
|
|
python3 tools/forgejo-sync.py --backfill
|
|
|
|
Show cache statistics::
|
|
|
|
python3 tools/forgejo-sync.py --stats
|
|
|
|
The cache file lives under ``tools/.cache/`` and is gitignored.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ensure we can import the sibling cache module regardless of cwd.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from _pipeline_cache import ( # noqa: E402
|
|
PipelineCache,
|
|
DEFAULT_CACHE_PATH,
|
|
)
|
|
|
|
|
|
def _load_token() -> str:
|
|
token = os.environ.get("GITEA_TOKEN")
|
|
if token:
|
|
return token.strip().strip('"').strip("'")
|
|
repo_root = Path(__file__).resolve().parent.parent
|
|
for envfile in (repo_root / ".devcontainer" / ".env", repo_root / ".env"):
|
|
if not envfile.is_file():
|
|
continue
|
|
for line in envfile.read_text().splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
k, _, v = line.partition("=")
|
|
if k.strip() == "GITEA_TOKEN":
|
|
return v.strip().strip('"').strip("'")
|
|
sys.exit(
|
|
"ERROR: GITEA_TOKEN not found. Set it in the environment "
|
|
"or in .devcontainer/.env (or .env)."
|
|
)
|
|
|
|
|
|
def _parse_args() -> argparse.Namespace:
|
|
p = argparse.ArgumentParser(
|
|
description="Sync the local Forgejo cache (SQLite at tools/.cache/forgejo.sqlite).",
|
|
)
|
|
p.add_argument(
|
|
"--full", action="store_true",
|
|
help="Truncate commits & pulls tables and resync from scratch. "
|
|
"Use after a force-push to master or to rebuild from clean state.",
|
|
)
|
|
p.add_argument(
|
|
"--backfill", action="store_true",
|
|
help="After syncing, fetch PR detail for every closed+merged PR missing "
|
|
"merged_by (one-shot per PR, permanent). Adds a few minutes on a "
|
|
"cold cache but makes subsequent reports offline-instant.",
|
|
)
|
|
p.add_argument(
|
|
"--stats", action="store_true",
|
|
help="Print cache statistics and exit; do not sync.",
|
|
)
|
|
p.add_argument(
|
|
"--cache", type=Path, default=DEFAULT_CACHE_PATH,
|
|
help=f"Cache file path (default: {DEFAULT_CACHE_PATH}).",
|
|
)
|
|
p.add_argument(
|
|
"--quiet", action="store_true",
|
|
help="Suppress progress logs; summary JSON is still printed.",
|
|
)
|
|
return p.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = _parse_args()
|
|
cache = PipelineCache.open(args.cache)
|
|
|
|
if args.stats:
|
|
print(json.dumps(cache.stats(), indent=2))
|
|
return
|
|
|
|
token = _load_token()
|
|
summary = cache.sync(
|
|
token,
|
|
full=args.full,
|
|
backfill_details=args.backfill,
|
|
progress=not args.quiet,
|
|
)
|
|
summary["cache"] = cache.stats()
|
|
print(json.dumps(summary, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|