#!/usr/bin/env bash # # opencode-builder.sh # # Script-driven orchestrator for the auto-agents system. # Queries the OpenCode server fleet, calculates worker pool gaps, # launches one-shot supervisors sequentially to spawn new workers, # and runs periodic health checks directly. # # Signal handling: # Ctrl+C once — finishes the in-flight request, then stops the loop # Ctrl+C twice — (within 2 seconds) kills the request and exits immediately # set -uo pipefail # ── Configuration (override via environment) ────────────────────────────── PORT="${OPENCODE_PORT:-4096}" HOST="${OPENCODE_HOST:-127.0.0.1}" BASE="http://${HOST}:${PORT}" HEALTH_TIMEOUT=60 MAX_IDLE_PER_MINUTE=10 # Worker pool sizing total_max_workers="${CA_MAX_PARALLEL_WORKERS:-4}" # Calculate per-type max workers (same logic as auto-agents) merge_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 7) / 8)}") imp_max_workers="${total_max_workers}" rev_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 1) / 2)}") # Health check intervals (seconds) DOOM_CHECK_INTERVAL=900 # 15 minutes DOOM_AGE_THRESHOLD=1800 # 30 minutes CONTINUE_BURST_WINDOW=60 # seconds CONTINUE_BURST_LIMIT=5 SLEEP_WHEN_IDLE=30 # seconds SUPERVISOR_TIMEOUT=300 # 5 minutes max for a one-shot supervisor # Script paths SCRIPT_DIR="/app/.opencode/skills/auto-agents-system/scripts" SESSION_LIST="npx --yes tsx ${SCRIPT_DIR}/session_list.ts" SESSION_FIND_PREFIX="npx --yes tsx ${SCRIPT_DIR}/session_find_by_prefix.ts" SESSION_START="npx --yes tsx ${SCRIPT_DIR}/session_start.ts" SESSION_MESSAGES="npx --yes tsx ${SCRIPT_DIR}/session_messages.ts" SESSION_DELETE="npx --yes tsx ${SCRIPT_DIR}/session_delete.ts" SESSION_STOP="npx --yes tsx ${SCRIPT_DIR}/session_stop.ts" # ── Internal state ──────────────────────────────────────────────────────── SERVER_PID="" CURL_PID="" STOP_LOOP=false LAST_SIGINT=0 OWN_SERVER=false N=0 IDLE_EVENTS_FILE="" # In-memory tracking for health checks declare -A QUESTION_COUNT declare -A CONTINUE_TIMES LAST_DOOM_CHECK=0 # ── Helpers ─────────────────────────────────────────────────────────────── die() { printf "ERROR: %s\n" "$*" >&2; exit 1; } log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; } check_opencode_running() { if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then return 0 else return 1 fi } cleanup() { if [[ -n "$IDLE_EVENTS_FILE" ]]; then rm -f "$IDLE_EVENTS_FILE" 2>/dev/null || true fi if $OWN_SERVER && [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then log "Stopping opencode server (PID $SERVER_PID)..." kill "$SERVER_PID" 2>/dev/null wait "$SERVER_PID" 2>/dev/null || true elif ! $OWN_SERVER; then log "Leaving existing OpenCode server running (not managed by this script)" fi } handle_sigint() { local now now=$(date +%s) # Second Ctrl+C within 2 seconds → force-quit if $STOP_LOOP && (( now - LAST_SIGINT <= 2 )); then printf "\nForce quit!\n" [[ -n "$CURL_PID" ]] && kill "$CURL_PID" 2>/dev/null || true cleanup exit 130 fi # First Ctrl+C → graceful stop after current request STOP_LOOP=true LAST_SIGINT=$now printf "\nWill stop after the current request completes. Ctrl+C again within 2s to force-quit.\n" } trap handle_sigint INT trap cleanup EXIT # ── Idle-event tracking ─────────────────────────────────────────────────── init_idle_tracking() { IDLE_EVENTS_FILE="/tmp/opencode-builder-idle-events-$$.txt" : > "$IDLE_EVENTS_FILE" 2>/dev/null || true } record_idle_event() { local now now=$(date +%s) echo "$now" >> "$IDLE_EVENTS_FILE" } count_idle_events_last_minute() { local now cutoff now=$(date +%s) cutoff=$((now - 60)) if [[ -f "$IDLE_EVENTS_FILE" ]]; then awk -v c="$cutoff" '$1 > c {count++} END {print count+0}' "$IDLE_EVENTS_FILE" 2>/dev/null else echo 0 fi } # ── Session lifecycle helpers ───────────────────────────────────────────── query_all_sessions() { $SESSION_LIST --server "${BASE}" 2>/dev/null } find_sessions_by_prefix() { local prefix="$1" local status_filter="${2:-all}" $SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null } create_session() { local title="$1" curl -sf -X POST "${BASE}/session" \ -H "Content-Type: application/json" \ -d "{\"title\":\"${title}\"}" 2>/dev/null } delete_session() { local sid="$1" $SESSION_DELETE --server "${BASE}" --session-id "$sid" --force >/dev/null 2>&1 || true } stop_session() { local sid="$1" $SESSION_STOP --server "${BASE}" --session-id "$sid" >/dev/null 2>&1 || true } # Send a message to a session and block until the LLM responds. # Captures the assistant's response to a temp file. send_message_capture() { local sid="$1" local body="$2" local out_file="$3" ( trap '' INT exec curl -sf -X POST "${BASE}/session/${sid}/message" \ -H "Content-Type: application/json" \ -d "$body" \ -w "\n" ) > "$out_file" 2>/dev/null & CURL_PID=$! while kill -0 "$CURL_PID" 2>/dev/null; do wait "$CURL_PID" 2>/dev/null || true done CURL_PID="" } # Poll GET /message until we see an assistant response or timeout get_messages_poll() { local sid="$1" local limit="${2:-10}" local timeout_secs="${3:-60}" local deadline deadline=$(($(date +%s) + timeout_secs)) while true; do local msgs msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit "$limit" 2>/dev/null) if [[ -n "$msgs" ]]; then # Check if there's an assistant message local has_assistant has_assistant=$(echo "$msgs" | jq -r 'map(select(.info.role == "assistant")) | length') if [[ "$has_assistant" -gt 0 ]]; then echo "$msgs" return 0 fi fi if [[ $(date +%s) -ge $deadline ]]; then echo "[]" return 1 fi sleep 2 done } # Extract the last assistant text from a messages JSON array extract_last_assistant_text() { local msgs="$1" echo "$msgs" | jq -r '[.[] | select(.info.role == "assistant")] | last | if . == null then "" else (.parts // [] | map(select(.type == "text")) | .[-1].text // "") end' } # Extract a JSON array from text (look for [ ... ] block) extract_json_array() { local text="$1" # Try to find a JSON array between [ and ] echo "$text" | grep -oP '\[\s*[^\]]*\]' | tail -n1 } # ── Question handling ───────────────────────────────────────────────────── get_pending_questions() { curl -sf "${BASE}/question" 2>/dev/null } answer_question() { local qid="$1" local text="$2" curl -sf -X POST "${BASE}/question/${qid}/answer" \ -H "Content-Type: application/json" \ -d "{\"body\":\"${text}\"}" >/dev/null 2>&1 } reject_question() { local qid="$1" curl -sf -X POST "${BASE}/question/${qid}/reject" >/dev/null 2>&1 } # ── Supervisor launcher ─────────────────────────────────────────────────── # Creates a temporary session, sends the supervisor prompt, blocks for the # response, extracts the JSON array of launched work numbers, and deletes # the temporary session. launch_supervisor() { local agent_name="$1" local tag="$2" local prompt_text="$3" local session_title="[${tag}] ${agent_name}" local tmp_prompt="/tmp/builder-prompt-${tag}-$$.txt" local tmp_response="/tmp/builder-response-${tag}-$$.txt" log "Creating supervisor session: ${session_title}" local session_json session_json=$(create_session "$session_title") local sid sid=$(echo "$session_json" | jq -r '.id') if [[ -z "$sid" || "$sid" == "null" ]]; then log "Failed to create session for ${tag}" return 1 fi # Write prompt to temp file printf '%s' "$prompt_text" > "$tmp_prompt" # Launch the agent asynchronously via prompt_async first log "Launching ${agent_name} in session ${sid}..." local start_body start_body=$(jq -nc --arg a "$agent_name" --rawfile t "$tmp_prompt" '{agent: $a, parts: [{type: "text", text: $t}]}') curl -sf -X POST "${BASE}/session/${sid}/prompt_async" \ -H "Content-Type: application/json" \ -d "$start_body" >/dev/null 2>&1 # Wait for the session to become idle (supervisor finished) local wait_deadline wait_deadline=$(($(date +%s) + SUPERVISOR_TIMEOUT)) local is_idle=false while [[ $(date +%s) -lt $wait_deadline ]]; do local status_json status_json=$(curl -sf "${BASE}/session/status" 2>/dev/null | jq -r --arg sid "$sid" '.[$sid].type // "idle"') if [[ "$status_json" == "idle" ]]; then is_idle=true break fi sleep 3 done if ! $is_idle; then log "Supervisor ${tag} timed out after ${SUPERVISOR_TIMEOUT}s" delete_session "$sid" rm -f "$tmp_prompt" "$tmp_response" return 1 fi # Fetch messages and extract the supervisor's response local msgs msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit 20 2>/dev/null) local response_text response_text=$(extract_last_assistant_text "$msgs") # Try to extract a JSON array from the response local launched_array launched_array=$(extract_json_array "$response_text") log "Supervisor ${tag} finished. Launched: ${launched_array:-none}" # Clean up temp session delete_session "$sid" rm -f "$tmp_prompt" "$tmp_response" # Return the extracted array echo "$launched_array" } # ── Fleet analysis ──────────────────────────────────────────────────────── # Returns the list of busy worker tags for a given prefix, one per line. get_busy_worker_tags() { local prefix="$1" find_sessions_by_prefix "$prefix" "busy" | jq -r '.[].tag // empty' } # Extract the PR/Issue number from a worker tag. # Tags look like: AUTO-IMP-PR-42, AUTO-IMP-ISSUE-42, AUTO-MRG-PR-42, AUTO-REV-PR-42 extract_work_number() { local tag="$1" local num num=$(echo "$tag" | grep -oP '(?<=(-PR-|-ISSUE-))\d+$') echo "$num" } # ── Duplicate worker guard ──────────────────────────────────────────────── deduplicate_workers() { local prefix="$1" local workers_json workers_json=$(find_sessions_by_prefix "$prefix" "busy") # Group by tag, keep only the most recently active for each tag local duplicates duplicates=$(echo "$workers_json" | jq -r ' group_by(.tag) | map(select(length > 1)) | map(sort_by(.last_active) | reverse | .[1:]) | flatten | .[].id ') if [[ -n "$duplicates" ]]; then while IFS= read -r dup_id; do [[ -n "$dup_id" ]] || continue log "Duplicate worker guard: deleting older session ${dup_id}" delete_session "$dup_id" done <<< "$duplicates" fi } # ── Disk cleanup ────────────────────────────────────────────────────────── cleanup_orphan_tmp_dirs() { local prefix="$1" local active_tags active_tags=$(find_sessions_by_prefix "$prefix" "all" | jq -r '.[].tag // empty' | sort -u) # Look for /tmp directories that match the agent pattern for this prefix # Expected pattern: /tmp/--/ # We can't know the exact agent name, so we look for directories with numbers # that might be identifiers, and cross-check against active tags. local tag_pattern="${prefix}" while IFS= read -r -d '' dir; do [[ -d "$dir" ]] || continue # Extract any numeric identifier from the directory basename local dir_id dir_id=$(basename "$dir" | grep -oP '\d+' | tail -n1) if [[ -n "$dir_id" ]]; then # Check if any active tag contains this number local is_active=false while IFS= read -r atag; do [[ -n "$atag" ]] || continue if [[ "$atag" == *"${dir_id}"* ]]; then is_active=true break fi done <<< "$active_tags" if ! $is_active; then log "Disk cleanup: removing orphan ${dir}" rm -rf "$dir" fi fi done < <(find /tmp -maxdepth 1 -type d -name "*${tag_pattern}*" -print0 2>/dev/null) } # ── Health checks ───────────────────────────────────────────────────────── run_health_cycle() { local now now=$(date +%s) local did_something=false # 1) Question handling local questions_json questions_json=$(get_pending_questions) if [[ -n "$questions_json" && "$questions_json" != "[]" && "$questions_json" != "null" ]]; then local qids sids qids=$(echo "$questions_json" | jq -r '.[].id') sids=$(echo "$questions_json" | jq -r '.[].sessionID') while IFS= read -r qid && IFS= read -r sid <&3; do [[ -n "$qid" ]] || continue [[ -n "$sid" ]] || continue local cnt="${QUESTION_COUNT[$sid]:-0}" if [[ "$cnt" -eq 0 ]]; then log "Question from ${sid} (first time) — answering with anti-question text" answer_question "$qid" "you are never under any circumstances to ask questions or pass along questions from subagents, use your best judgement and do not ask questions again" QUESTION_COUNT[$sid]=1 else log "Question from ${sid} (repeat) — deleting session" delete_session "$sid" unset QUESTION_COUNT[$sid] fi did_something=true done <<< "$qids" 3<<< "$sids" fi # 2) Idle worker check local all_prefixes=("AUTO-IMP" "AUTO-MRG" "AUTO-REV") for prefix in "${all_prefixes[@]}"; do local idle_workers idle_workers=$(find_sessions_by_prefix "$prefix" "idle" | jq -r '.[].id // empty') while IFS= read -r sid; do [[ -n "$sid" ]] || continue local msgs msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit 10 2>/dev/null) if [[ -z "$msgs" || "$msgs" == "[]" ]]; then log "Idle worker ${sid} has no messages — deleting" delete_session "$sid" did_something=true continue fi # Check if it appears partially done (last assistant message suggests more work) local last_text last_text=$(extract_last_assistant_text "$msgs") # Heuristic: if the last text contains completion signals, delete it if echo "$last_text" | grep -qiE '(done|finished|complete|exiting|all tests pass|committed|merged|review submitted)'; then log "Idle worker ${sid} appears finished — deleting" delete_session "$sid" did_something=true continue fi # Otherwise, send continue and track local now_ms now_ms=$(date +%s) local times="${CONTINUE_TIMES[$sid]:-}" # Keep only timestamps within the burst window local cutoff=$((now_ms - CONTINUE_BURST_WINDOW)) local new_times="" local count=0 if [[ -n "$times" ]]; then while IFS= read -r ts; do [[ -n "$ts" ]] || continue if [[ "$ts" -gt "$cutoff" ]]; then new_times="${new_times}${ts}\n" ((count++)) fi done <<< "$times" fi new_times="${new_times}${now_ms}\n" ((count++)) CONTINUE_TIMES[$sid]="$new_times" if [[ "$count" -gt $CONTINUE_BURST_LIMIT ]]; then log "Idle worker ${sid} received ${count} continues in ${CONTINUE_BURST_WINDOW}s — deleting" delete_session "$sid" unset CONTINUE_TIMES[$sid] did_something=true else log "Idle worker ${sid} — sending continue" local cont_body cont_body=$(jq -nc '{agent: "", parts: [{type: "text", text: "continue"}]}') send_message_capture "$sid" "$cont_body" "/tmp/continue-${sid}.txt" did_something=true fi done <<< "$idle_workers" done # 3) Doom-loop check (every 15 minutes) if [[ $((now - LAST_DOOM_CHECK)) -ge $DOOM_CHECK_INTERVAL ]]; then LAST_DOOM_CHECK=$now for prefix in "${all_prefixes[@]}"; do local busy_workers busy_workers=$(find_sessions_by_prefix "$prefix" "busy" | jq -r '.[] | select(.last_active < ('"$now"' * 1000 - '"$((DOOM_AGE_THRESHOLD * 1000))"')) | .id // empty') while IFS= read -r sid; do [[ -n "$sid" ]] || continue log "Doom-loop check for ${sid} (busy >30 min)" local eval_tag="BUILDER-HEALTH-${sid: -6}" local eval_title="[${eval_tag}] worker-health-evaluator" local eval_prompt="session_id: ${sid} Evaluate the health of this session." local eval_session_json eval_session_json=$(create_session "$eval_title") local eval_sid eval_sid=$(echo "$eval_session_json" | jq -r '.id') if [[ -n "$eval_sid" && "$eval_sid" != "null" ]]; then local eval_tmp="/tmp/builder-prompt-${eval_tag}-$$.txt" printf '%s' "$eval_prompt" > "$eval_tmp" local eval_body eval_body=$(jq -nc --arg a "worker-health-evaluator" --rawfile t "$eval_tmp" '{agent: $a, parts: [{type: "text", text: $t}]}') curl -sf -X POST "${BASE}/session/${eval_sid}/prompt_async" -H "Content-Type: application/json" -d "$eval_body" >/dev/null 2>&1 local eval_deadline eval_deadline=$(($(date +%s) + 120)) while [[ $(date +%s) -lt $eval_deadline ]]; do local eval_status eval_status=$(curl -sf "${BASE}/session/status" 2>/dev/null | jq -r --arg sid "$eval_sid" '.[$sid].type // "idle"') [[ "$eval_status" == "idle" ]] && break sleep 2 done local eval_msgs eval_msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$eval_sid" --limit 10 2>/dev/null) local eval_text eval_text=$(extract_last_assistant_text "$eval_msgs") delete_session "$eval_sid" rm -f "$eval_tmp" local is_unrecoverable is_unrecoverable=$(echo "$eval_text" | jq -r '.unrecoverable // false' 2>/dev/null || echo "false") if [[ "$is_unrecoverable" == "true" ]]; then log "Session ${sid} is unrecoverable — deleting" delete_session "$sid" did_something=true fi fi done <<< "$busy_workers" done fi if $did_something; then return 0 else return 1 fi } # ── Preflight checks ───────────────────────────────────────────────────── for cmd in curl jq opencode npx; do command -v "$cmd" &>/dev/null || die "'$cmd' is required but not found in PATH" done # ── Start the opencode server ───────────────────────────────────────────── if check_opencode_running; then echo "┌─────────────────────────────────────────────────────────────────┐" echo "│ ⚠️ WARNING: OpenCode server already running on ${HOST}:${PORT}" echo "│ 🔗 Connecting to existing server instead of starting new one" echo "│ 💡 The script will NOT manage this server's lifecycle" echo "└─────────────────────────────────────────────────────────────────┘" echo "" log "Using existing OpenCode server at ${HOST}:${PORT}" OWN_SERVER=false else log "Starting opencode server on ${HOST}:${PORT}..." opencode serve --port "$PORT" --hostname "$HOST" & SERVER_PID=$! OWN_SERVER=true log "Waiting for server to become healthy (up to ${HEALTH_TIMEOUT}s)..." healthy=false for (( i = 1; i <= HEALTH_TIMEOUT; i++ )); do if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then healthy=true break fi kill -0 "$SERVER_PID" 2>/dev/null || die "Server process exited unexpectedly" sleep 1 done $healthy || die "Server did not become healthy within ${HEALTH_TIMEOUT}s" log "Server is ready." fi # ── Main orchestration loop ─────────────────────────────────────────────── log "Starting orchestration loop." log "Worker pools: merge=${merge_max_workers} imp=${imp_max_workers} rev=${rev_max_workers}" while ! $STOP_LOOP; do N=$((N + 1)) # Check idle-event rate in the last minute idle_count=$(count_idle_events_last_minute) if (( idle_count > MAX_IDLE_PER_MINUTE )); then log "── iteration #${N} ── TOO MANY IDLE EVENTS (${idle_count}) — skipping cycle" sleep 5 continue fi log "── iteration #${N} ──" local did_anything=false # ── Step 1: Query fleet ─────────────────────────────────────────────── local all_sessions_json all_sessions_json=$(query_all_sessions) if [[ -z "$all_sessions_json" ]]; then log "No sessions found." fi # ── Step 2: Per-type analysis ───────────────────────────────────────── # Type definitions: prefix, supervisor agent name, max workers local types=( "AUTO-IMP:implementation-supervisor:${imp_max_workers}" "AUTO-MRG:pr-merge-supervisor:${merge_max_workers}" "AUTO-REV:pr-review-supervisor:${rev_max_workers}" ) for type_spec in "${types[@]}"; do IFS=':' read -r prefix supervisor_name max <<< "$type_spec" # Duplicate guard deduplicate_workers "$prefix" # Disk cleanup cleanup_orphan_tmp_dirs "$prefix" # Count busy workers and collect active work numbers local busy_tags busy_tags=$(get_busy_worker_tags "$prefix") local busy_count=0 local exclude_numbers="" while IFS= read -r tag; do [[ -n "$tag" ]] || continue ((busy_count++)) local num num=$(extract_work_number "$tag") if [[ -n "$num" ]]; then exclude_numbers="${exclude_numbers}${num}," fi done <<< "$busy_tags" # Remove trailing comma exclude_numbers="${exclude_numbers%,}" local available_slots=$((max - busy_count)) if [[ "$available_slots" -le 0 ]]; then log "Pool ${prefix} is full (${busy_count}/${max})" continue fi log "Pool ${prefix}: ${busy_count}/${max} busy, spawning up to ${available_slots} workers" did_anything=true # Build supervisor prompt local prompt_text prompt_text=$(cat <