674 lines
24 KiB
Bash
Executable File
674 lines
24 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# opencode-builder.sh
|
|
#
|
|
# Script-driven orchestrator for the auto-agents system.
|
|
# Queries the OpenCode server fleet, calculates worker pool gaps,
|
|
# launches one-shot supervisors sequentially to spawn new workers,
|
|
# and runs periodic health checks directly.
|
|
#
|
|
# Signal handling:
|
|
# Ctrl+C once — finishes the in-flight request, then stops the loop
|
|
# Ctrl+C twice — (within 2 seconds) kills the request and exits immediately
|
|
#
|
|
|
|
set -uo pipefail
|
|
|
|
# ── Configuration (override via environment) ──────────────────────────────
|
|
PORT="${OPENCODE_PORT:-4096}"
|
|
HOST="${OPENCODE_HOST:-127.0.0.1}"
|
|
BASE="http://${HOST}:${PORT}"
|
|
HEALTH_TIMEOUT=60
|
|
MAX_IDLE_PER_MINUTE=10
|
|
|
|
# Worker pool sizing
|
|
total_max_workers="${CA_MAX_PARALLEL_WORKERS:-4}"
|
|
# Calculate per-type max workers (same logic as auto-agents)
|
|
merge_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 7) / 8)}")
|
|
imp_max_workers="${total_max_workers}"
|
|
rev_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 1) / 2)}")
|
|
|
|
# Health check intervals (seconds)
|
|
DOOM_CHECK_INTERVAL=900 # 15 minutes
|
|
DOOM_AGE_THRESHOLD=1800 # 30 minutes
|
|
CONTINUE_BURST_WINDOW=60 # seconds
|
|
CONTINUE_BURST_LIMIT=5
|
|
SLEEP_WHEN_IDLE=30 # seconds
|
|
SUPERVISOR_TIMEOUT=1200 # 20 minutes max for a one-shot supervisor
|
|
|
|
# Script paths
|
|
SCRIPT_DIR="/app/.opencode/skills/auto-agents-system/scripts"
|
|
SESSION_LIST="npx --yes tsx ${SCRIPT_DIR}/session_list.ts"
|
|
SESSION_FIND_PREFIX="npx --yes tsx ${SCRIPT_DIR}/session_find_by_prefix.ts"
|
|
SESSION_START="npx --yes tsx ${SCRIPT_DIR}/session_start.ts"
|
|
SESSION_MESSAGES="npx --yes tsx ${SCRIPT_DIR}/session_messages.ts"
|
|
SESSION_DELETE="npx --yes tsx ${SCRIPT_DIR}/session_delete.ts"
|
|
SESSION_STOP="npx --yes tsx ${SCRIPT_DIR}/session_stop.ts"
|
|
|
|
# ── Internal state ────────────────────────────────────────────────────────
|
|
SERVER_PID=""
|
|
CURL_PID=""
|
|
STOP_LOOP=false
|
|
LAST_SIGINT=0
|
|
OWN_SERVER=false
|
|
N=0
|
|
IDLE_EVENTS_FILE=""
|
|
|
|
# In-memory tracking for health checks
|
|
declare -A QUESTION_COUNT
|
|
declare -A CONTINUE_TIMES
|
|
LAST_DOOM_CHECK=0
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────────
|
|
die() { printf "ERROR: %s\n" "$*" >&2; exit 1; }
|
|
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
|
|
|
check_opencode_running() {
|
|
if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
cleanup() {
|
|
if [[ -n "$IDLE_EVENTS_FILE" ]]; then
|
|
rm -f "$IDLE_EVENTS_FILE" 2>/dev/null || true
|
|
fi
|
|
if $OWN_SERVER && [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
|
|
log "Stopping opencode server (PID $SERVER_PID)..."
|
|
kill "$SERVER_PID" 2>/dev/null
|
|
wait "$SERVER_PID" 2>/dev/null || true
|
|
elif ! $OWN_SERVER; then
|
|
log "Leaving existing OpenCode server running (not managed by this script)"
|
|
fi
|
|
}
|
|
|
|
handle_sigint() {
|
|
local now
|
|
now=$(date +%s)
|
|
|
|
# Second Ctrl+C within 2 seconds → force-quit
|
|
if $STOP_LOOP && (( now - LAST_SIGINT <= 2 )); then
|
|
printf "\nForce quit!\n"
|
|
[[ -n "$CURL_PID" ]] && kill "$CURL_PID" 2>/dev/null || true
|
|
cleanup
|
|
exit 130
|
|
fi
|
|
|
|
# First Ctrl+C → graceful stop after current request
|
|
STOP_LOOP=true
|
|
LAST_SIGINT=$now
|
|
printf "\nWill stop after the current request completes. Ctrl+C again within 2s to force-quit.\n"
|
|
}
|
|
|
|
trap handle_sigint INT
|
|
trap cleanup EXIT
|
|
|
|
# ── Idle-event tracking ───────────────────────────────────────────────────
|
|
init_idle_tracking() {
|
|
IDLE_EVENTS_FILE="/tmp/opencode-builder-idle-events-$$.txt"
|
|
: > "$IDLE_EVENTS_FILE" 2>/dev/null || true
|
|
}
|
|
|
|
record_idle_event() {
|
|
local now
|
|
now=$(date +%s)
|
|
echo "$now" >> "$IDLE_EVENTS_FILE"
|
|
}
|
|
|
|
count_idle_events_last_minute() {
|
|
local now cutoff
|
|
now=$(date +%s)
|
|
cutoff=$((now - 60))
|
|
if [[ -f "$IDLE_EVENTS_FILE" ]]; then
|
|
awk -v c="$cutoff" '$1 > c {count++} END {print count+0}' "$IDLE_EVENTS_FILE" 2>/dev/null
|
|
else
|
|
echo 0
|
|
fi
|
|
}
|
|
|
|
# ── Session lifecycle helpers ─────────────────────────────────────────────
|
|
query_all_sessions() {
|
|
$SESSION_LIST --server "${BASE}" 2>/dev/null
|
|
}
|
|
|
|
find_sessions_by_prefix() {
|
|
local prefix="$1"
|
|
local status_filter="${2:-all}"
|
|
$SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null
|
|
}
|
|
|
|
create_session() {
|
|
local title="$1"
|
|
curl -sf -X POST "${BASE}/session" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"title\":\"${title}\"}" 2>/dev/null
|
|
}
|
|
|
|
delete_session() {
|
|
local sid="$1"
|
|
$SESSION_DELETE --server "${BASE}" --session-id "$sid" --force >/dev/null 2>&1 || true
|
|
}
|
|
|
|
stop_session() {
|
|
local sid="$1"
|
|
$SESSION_STOP --server "${BASE}" --session-id "$sid" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Send a message to a session and block until the LLM responds.
|
|
# Captures the assistant's response to a temp file.
|
|
send_message_capture() {
|
|
local sid="$1"
|
|
local body="$2"
|
|
local out_file="$3"
|
|
|
|
( trap '' INT
|
|
exec curl -sf -X POST "${BASE}/session/${sid}/message" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$body" \
|
|
-w "\n"
|
|
) > "$out_file" 2>/dev/null &
|
|
CURL_PID=$!
|
|
|
|
while kill -0 "$CURL_PID" 2>/dev/null; do
|
|
wait "$CURL_PID" 2>/dev/null || true
|
|
done
|
|
CURL_PID=""
|
|
}
|
|
|
|
# Poll GET /message until we see an assistant response or timeout
|
|
get_messages_poll() {
|
|
local sid="$1"
|
|
local limit="${2:-10}"
|
|
local timeout_secs="${3:-60}"
|
|
local deadline
|
|
deadline=$(($(date +%s) + timeout_secs))
|
|
|
|
while true; do
|
|
local msgs
|
|
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit "$limit" 2>/dev/null)
|
|
if [[ -n "$msgs" ]]; then
|
|
# Check if there's an assistant message
|
|
local has_assistant
|
|
has_assistant=$(echo "$msgs" | jq -r 'map(select(.info.role == "assistant")) | length')
|
|
if [[ "$has_assistant" -gt 0 ]]; then
|
|
echo "$msgs"
|
|
return 0
|
|
fi
|
|
fi
|
|
if [[ $(date +%s) -ge $deadline ]]; then
|
|
echo "[]"
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
}
|
|
|
|
# Extract the last assistant text from a messages JSON array
|
|
extract_last_assistant_text() {
|
|
local msgs="$1"
|
|
echo "$msgs" | jq -r '[.[] | select(.info.role == "assistant")] | last | if . == null then "" else (.parts // [] | map(select(.type == "text")) | .[-1].text // "") end'
|
|
}
|
|
|
|
# Extract a JSON array from text (look for [ ... ] block)
|
|
extract_json_array() {
|
|
local text="$1"
|
|
# Try to find a JSON array between [ and ]
|
|
echo "$text" | grep -oP '\[\s*[^\]]*\]' | tail -n1
|
|
}
|
|
|
|
# ── Question handling ─────────────────────────────────────────────────────
|
|
get_pending_questions() {
|
|
curl -sf "${BASE}/question" 2>/dev/null
|
|
}
|
|
|
|
answer_question() {
|
|
local qid="$1"
|
|
local text="$2"
|
|
curl -sf -X POST "${BASE}/question/${qid}/answer" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"body\":\"${text}\"}" >/dev/null 2>&1
|
|
}
|
|
|
|
reject_question() {
|
|
local qid="$1"
|
|
curl -sf -X POST "${BASE}/question/${qid}/reject" >/dev/null 2>&1
|
|
}
|
|
|
|
# ── Supervisor launcher ───────────────────────────────────────────────────
|
|
# Creates a temporary session, sends the supervisor prompt, blocks for the
|
|
# response, extracts the JSON array of launched work numbers, and deletes
|
|
# the temporary session.
|
|
launch_supervisor() {
|
|
local agent_name="$1"
|
|
local tag="$2"
|
|
local prompt_text="$3"
|
|
local session_title="[${tag}] ${agent_name}"
|
|
local tmp_prompt="/tmp/builder-prompt-${tag}-$$.txt"
|
|
local tmp_response="/tmp/builder-response-${tag}-$$.txt"
|
|
|
|
log "Creating supervisor session: ${session_title}"
|
|
local session_json
|
|
session_json=$(create_session "$session_title")
|
|
local sid
|
|
sid=$(echo "$session_json" | jq -r '.id')
|
|
if [[ -z "$sid" || "$sid" == "null" ]]; then
|
|
log "Failed to create session for ${tag}"
|
|
return 1
|
|
fi
|
|
|
|
# Write prompt to temp file
|
|
printf '%s' "$prompt_text" > "$tmp_prompt"
|
|
|
|
# Launch the agent asynchronously via prompt_async first
|
|
log "Launching ${agent_name} in session ${sid}..."
|
|
local start_body
|
|
start_body=$(jq -nc --arg a "$agent_name" --rawfile t "$tmp_prompt" '{agent: $a, parts: [{type: "text", text: $t}]}')
|
|
curl -sf -X POST "${BASE}/session/${sid}/prompt_async" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$start_body" >/dev/null 2>&1
|
|
|
|
# Wait for the session to become idle (supervisor finished)
|
|
local wait_deadline
|
|
wait_deadline=$(($(date +%s) + SUPERVISOR_TIMEOUT))
|
|
local is_idle=false
|
|
while [[ $(date +%s) -lt $wait_deadline ]]; do
|
|
local status_json
|
|
status_json=$(curl -sf "${BASE}/session/status" 2>/dev/null | jq -r --arg sid "$sid" '.[$sid].type // "idle"')
|
|
if [[ "$status_json" == "idle" ]]; then
|
|
is_idle=true
|
|
break
|
|
fi
|
|
sleep 3
|
|
done
|
|
|
|
if ! $is_idle; then
|
|
log "Supervisor ${tag} timed out after ${SUPERVISOR_TIMEOUT}s"
|
|
delete_session "$sid"
|
|
rm -f "$tmp_prompt" "$tmp_response"
|
|
return 1
|
|
fi
|
|
|
|
# Fetch messages and extract the supervisor's response
|
|
local msgs
|
|
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit 20 2>/dev/null)
|
|
local response_text
|
|
response_text=$(extract_last_assistant_text "$msgs")
|
|
|
|
# Try to extract a JSON array from the response
|
|
local launched_array
|
|
launched_array=$(extract_json_array "$response_text")
|
|
|
|
log "Supervisor ${tag} finished. Launched: ${launched_array:-none}"
|
|
|
|
# Clean up temp session
|
|
delete_session "$sid"
|
|
rm -f "$tmp_prompt" "$tmp_response"
|
|
|
|
# Return the extracted array
|
|
echo "$launched_array"
|
|
}
|
|
|
|
# ── Fleet analysis ────────────────────────────────────────────────────────
|
|
# Returns the list of busy worker tags for a given prefix, one per line.
|
|
get_busy_worker_tags() {
|
|
local prefix="$1"
|
|
find_sessions_by_prefix "$prefix" "busy" | jq -r '.[].tag // empty'
|
|
}
|
|
|
|
# Extract the PR/Issue number from a worker tag.
|
|
# Tags look like: AUTO-IMP-PR-42, AUTO-IMP-ISSUE-42, AUTO-MRG-PR-42, AUTO-REV-PR-42
|
|
extract_work_number() {
|
|
local tag="$1"
|
|
echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p'
|
|
}
|
|
|
|
# ── Duplicate worker guard ────────────────────────────────────────────────
|
|
deduplicate_workers() {
|
|
local prefix="$1"
|
|
local workers_json
|
|
workers_json=$(find_sessions_by_prefix "$prefix" "busy")
|
|
# Group by tag, keep only the most recently active for each tag
|
|
local duplicates
|
|
duplicates=$(echo "$workers_json" | jq -r '
|
|
group_by(.tag)
|
|
| map(select(length > 1))
|
|
| map(sort_by(.last_active) | reverse | .[1:])
|
|
| flatten
|
|
| .[].id
|
|
')
|
|
if [[ -n "$duplicates" ]]; then
|
|
while IFS= read -r dup_id; do
|
|
[[ -n "$dup_id" ]] || continue
|
|
log "Duplicate worker guard: deleting older session ${dup_id}"
|
|
delete_session "$dup_id"
|
|
done <<< "$duplicates"
|
|
fi
|
|
}
|
|
|
|
# ── Disk cleanup ──────────────────────────────────────────────────────────
|
|
cleanup_orphan_tmp_dirs() {
|
|
local prefix="$1"
|
|
local active_tags
|
|
active_tags=$(find_sessions_by_prefix "$prefix" "all" | jq -r '.[].tag // empty' | sort -u)
|
|
|
|
# Look for /tmp directories that match the agent pattern for this prefix
|
|
# Expected pattern: /tmp/<agent_name>-<identifier>-<timestamp>/
|
|
# We can't know the exact agent name, so we look for directories with numbers
|
|
# that might be identifiers, and cross-check against active tags.
|
|
local tag_pattern="${prefix}"
|
|
while IFS= read -r -d '' dir; do
|
|
[[ -d "$dir" ]] || continue
|
|
# Extract any numeric identifier from the directory basename
|
|
local dir_id
|
|
dir_id=$(basename "$dir" | grep -oP '\d+' | tail -n1)
|
|
if [[ -n "$dir_id" ]]; then
|
|
# Check if any active tag contains this number
|
|
local is_active=false
|
|
while IFS= read -r atag; do
|
|
[[ -n "$atag" ]] || continue
|
|
if [[ "$atag" == *"${dir_id}"* ]]; then
|
|
is_active=true
|
|
break
|
|
fi
|
|
done <<< "$active_tags"
|
|
if ! $is_active; then
|
|
log "Disk cleanup: removing orphan ${dir}"
|
|
rm -rf "$dir"
|
|
fi
|
|
fi
|
|
done < <(find /tmp -maxdepth 1 -type d -name "*${tag_pattern}*" -print0 2>/dev/null)
|
|
}
|
|
|
|
# ── Health checks ─────────────────────────────────────────────────────────
|
|
run_health_cycle() {
|
|
local now
|
|
now=$(date +%s)
|
|
local did_something=false
|
|
|
|
# 1) Question handling
|
|
local questions_json
|
|
questions_json=$(get_pending_questions)
|
|
if [[ -n "$questions_json" && "$questions_json" != "[]" && "$questions_json" != "null" ]]; then
|
|
local qids sids
|
|
qids=$(echo "$questions_json" | jq -r '.[].id')
|
|
sids=$(echo "$questions_json" | jq -r '.[].sessionID')
|
|
while IFS= read -r qid && IFS= read -r sid <&3; do
|
|
[[ -n "$qid" ]] || continue
|
|
[[ -n "$sid" ]] || continue
|
|
local cnt="${QUESTION_COUNT[$sid]:-0}"
|
|
if [[ "$cnt" -eq 0 ]]; then
|
|
log "Question from ${sid} (first time) — answering with anti-question text"
|
|
answer_question "$qid" "you are never under any circumstances to ask questions or pass along questions from subagents, use your best judgement and do not ask questions again"
|
|
QUESTION_COUNT[$sid]=1
|
|
else
|
|
log "Question from ${sid} (repeat) — deleting session"
|
|
delete_session "$sid"
|
|
unset QUESTION_COUNT[$sid]
|
|
fi
|
|
did_something=true
|
|
done <<< "$qids" 3<<< "$sids"
|
|
fi
|
|
|
|
# 2) Idle worker check
|
|
local all_prefixes=("AUTO-IMP" "AUTO-MRG" "AUTO-REV")
|
|
for prefix in "${all_prefixes[@]}"; do
|
|
local idle_workers
|
|
idle_workers=$(find_sessions_by_prefix "$prefix" "idle" | jq -r '.[].id // empty')
|
|
while IFS= read -r sid; do
|
|
[[ -n "$sid" ]] || continue
|
|
local msgs
|
|
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit 10 2>/dev/null)
|
|
if [[ -z "$msgs" || "$msgs" == "[]" ]]; then
|
|
log "Idle worker ${sid} has no messages — deleting"
|
|
delete_session "$sid"
|
|
did_something=true
|
|
continue
|
|
fi
|
|
# Check if it appears partially done (last assistant message suggests more work)
|
|
local last_text
|
|
last_text=$(extract_last_assistant_text "$msgs")
|
|
# Heuristic: if the last text contains completion signals, delete it
|
|
if echo "$last_text" | grep -qiE '(done|finished|complete|exiting|all tests pass|committed|merged|review submitted)'; then
|
|
log "Idle worker ${sid} appears finished — deleting"
|
|
delete_session "$sid"
|
|
did_something=true
|
|
continue
|
|
fi
|
|
# Otherwise, send continue and track
|
|
local now_ms
|
|
now_ms=$(date +%s)
|
|
local times_str="${CONTINUE_TIMES[$sid]:-}"
|
|
local cutoff=$((now_ms - CONTINUE_BURST_WINDOW))
|
|
# Store as space-delimited timestamps to avoid newline parsing issues
|
|
local new_times_arr=()
|
|
local count=0
|
|
if [[ -n "$times_str" ]]; then
|
|
for ts in $times_str; do
|
|
[[ -n "$ts" ]] || continue
|
|
if (( ts > cutoff )); then
|
|
new_times_arr+=("$ts")
|
|
((count++))
|
|
fi
|
|
done
|
|
fi
|
|
new_times_arr+=("$now_ms")
|
|
((count++))
|
|
CONTINUE_TIMES[$sid]=$(printf '%s ' "${new_times_arr[@]}")
|
|
if [[ "$count" -gt $CONTINUE_BURST_LIMIT ]]; then
|
|
log "Idle worker ${sid} received ${count} continues in ${CONTINUE_BURST_WINDOW}s — deleting"
|
|
delete_session "$sid"
|
|
unset CONTINUE_TIMES[$sid]
|
|
did_something=true
|
|
else
|
|
log "Idle worker ${sid} — sending continue"
|
|
local cont_body
|
|
cont_body=$(jq -nc '{agent: "", parts: [{type: "text", text: "continue"}]}')
|
|
send_message_capture "$sid" "$cont_body" "/tmp/continue-${sid}.txt"
|
|
did_something=true
|
|
fi
|
|
done <<< "$idle_workers"
|
|
done
|
|
|
|
# 3) Doom-loop check (every 15 minutes)
|
|
if [[ $((now - LAST_DOOM_CHECK)) -ge $DOOM_CHECK_INTERVAL ]]; then
|
|
LAST_DOOM_CHECK=$now
|
|
for prefix in "${all_prefixes[@]}"; do
|
|
local busy_workers
|
|
busy_workers=$(find_sessions_by_prefix "$prefix" "busy" | jq -r '.[] | select(.last_active < ('"$now"' * 1000 - '"$((DOOM_AGE_THRESHOLD * 1000))"')) | .id // empty')
|
|
while IFS= read -r sid; do
|
|
[[ -n "$sid" ]] || continue
|
|
log "Doom-loop check for ${sid} (busy >30 min)"
|
|
local eval_tag="BUILDER-HEALTH-${sid: -6}"
|
|
local eval_title="[${eval_tag}] worker-health-evaluator"
|
|
local eval_prompt="session_id: ${sid}
|
|
|
|
Evaluate the health of this session."
|
|
local eval_session_json
|
|
eval_session_json=$(create_session "$eval_title")
|
|
local eval_sid
|
|
eval_sid=$(echo "$eval_session_json" | jq -r '.id')
|
|
if [[ -n "$eval_sid" && "$eval_sid" != "null" ]]; then
|
|
local eval_tmp="/tmp/builder-prompt-${eval_tag}-$$.txt"
|
|
printf '%s' "$eval_prompt" > "$eval_tmp"
|
|
local eval_body
|
|
eval_body=$(jq -nc --arg a "worker-health-evaluator" --rawfile t "$eval_tmp" '{agent: $a, parts: [{type: "text", text: $t}]}')
|
|
curl -sf -X POST "${BASE}/session/${eval_sid}/prompt_async" -H "Content-Type: application/json" -d "$eval_body" >/dev/null 2>&1
|
|
local eval_deadline
|
|
eval_deadline=$(($(date +%s) + 120))
|
|
while [[ $(date +%s) -lt $eval_deadline ]]; do
|
|
local eval_status
|
|
eval_status=$(curl -sf "${BASE}/session/status" 2>/dev/null | jq -r --arg sid "$eval_sid" '.[$sid].type // "idle"')
|
|
[[ "$eval_status" == "idle" ]] && break
|
|
sleep 2
|
|
done
|
|
local eval_msgs
|
|
eval_msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$eval_sid" --limit 10 2>/dev/null)
|
|
local eval_text
|
|
eval_text=$(extract_last_assistant_text "$eval_msgs")
|
|
delete_session "$eval_sid"
|
|
rm -f "$eval_tmp"
|
|
local is_unrecoverable
|
|
is_unrecoverable=$(echo "$eval_text" | jq -r '.unrecoverable // false' 2>/dev/null || echo "false")
|
|
if [[ "$is_unrecoverable" == "true" ]]; then
|
|
log "Session ${sid} is unrecoverable — deleting"
|
|
delete_session "$sid"
|
|
did_something=true
|
|
fi
|
|
fi
|
|
done <<< "$busy_workers"
|
|
done
|
|
fi
|
|
|
|
if $did_something; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ── Preflight checks ─────────────────────────────────────────────────────
|
|
for cmd in curl jq opencode npx; do
|
|
command -v "$cmd" &>/dev/null || die "'$cmd' is required but not found in PATH"
|
|
done
|
|
|
|
# ── Start the opencode server ─────────────────────────────────────────────
|
|
if check_opencode_running; then
|
|
echo "┌─────────────────────────────────────────────────────────────────┐"
|
|
echo "│ ⚠️ WARNING: OpenCode server already running on ${HOST}:${PORT}"
|
|
echo "│ 🔗 Connecting to existing server instead of starting new one"
|
|
echo "│ 💡 The script will NOT manage this server's lifecycle"
|
|
echo "└─────────────────────────────────────────────────────────────────┘"
|
|
echo ""
|
|
log "Using existing OpenCode server at ${HOST}:${PORT}"
|
|
OWN_SERVER=false
|
|
else
|
|
log "Starting opencode server on ${HOST}:${PORT}..."
|
|
opencode serve --port "$PORT" --hostname "$HOST" &
|
|
SERVER_PID=$!
|
|
OWN_SERVER=true
|
|
|
|
log "Waiting for server to become healthy (up to ${HEALTH_TIMEOUT}s)..."
|
|
healthy=false
|
|
for (( i = 1; i <= HEALTH_TIMEOUT; i++ )); do
|
|
if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then
|
|
healthy=true
|
|
break
|
|
fi
|
|
kill -0 "$SERVER_PID" 2>/dev/null || die "Server process exited unexpectedly"
|
|
sleep 1
|
|
done
|
|
$healthy || die "Server did not become healthy within ${HEALTH_TIMEOUT}s"
|
|
log "Server is ready."
|
|
fi
|
|
|
|
# ── Main orchestration loop ───────────────────────────────────────────────
|
|
main_loop() {
|
|
log "Starting orchestration loop."
|
|
log "Worker pools: merge=${merge_max_workers} imp=${imp_max_workers} rev=${rev_max_workers}"
|
|
|
|
while ! $STOP_LOOP; do
|
|
N=$((N + 1))
|
|
|
|
# Check idle-event rate in the last minute
|
|
idle_count=$(count_idle_events_last_minute)
|
|
if (( idle_count > MAX_IDLE_PER_MINUTE )); then
|
|
log "── iteration #${N} ── TOO MANY IDLE EVENTS (${idle_count}) — skipping cycle"
|
|
sleep 5
|
|
continue
|
|
fi
|
|
|
|
log "── iteration #${N} ──"
|
|
local did_anything=false
|
|
|
|
# ── Step 1: Query fleet ───────────────────────────────────────────────
|
|
local all_sessions_json
|
|
all_sessions_json=$(query_all_sessions)
|
|
if [[ -z "$all_sessions_json" ]]; then
|
|
log "No sessions found."
|
|
fi
|
|
|
|
# ── Step 2: Per-type analysis ─────────────────────────────────────────
|
|
# Type definitions: prefix, supervisor agent name, max workers
|
|
local types=(
|
|
"AUTO-IMP:implementation-supervisor:${imp_max_workers}"
|
|
"AUTO-MRG:pr-merge-supervisor:${merge_max_workers}"
|
|
"AUTO-REV:pr-review-supervisor:${rev_max_workers}"
|
|
)
|
|
|
|
for type_spec in "${types[@]}"; do
|
|
IFS=':' read -r prefix supervisor_name max <<< "$type_spec"
|
|
|
|
# Duplicate guard
|
|
deduplicate_workers "$prefix"
|
|
|
|
# Disk cleanup
|
|
cleanup_orphan_tmp_dirs "$prefix"
|
|
|
|
# Count busy workers and collect active work numbers
|
|
local busy_tags
|
|
busy_tags=$(get_busy_worker_tags "$prefix")
|
|
local busy_count=0
|
|
local exclude_numbers=""
|
|
while IFS= read -r tag; do
|
|
[[ -n "$tag" ]] || continue
|
|
((busy_count++))
|
|
local num
|
|
num=$(extract_work_number "$tag")
|
|
if [[ -n "$num" ]]; then
|
|
exclude_numbers="${exclude_numbers}${num},"
|
|
fi
|
|
done <<< "$busy_tags"
|
|
# Remove trailing comma
|
|
exclude_numbers="${exclude_numbers%,}"
|
|
|
|
local available_slots=$((max - busy_count))
|
|
if [[ "$available_slots" -le 0 ]]; then
|
|
log "Pool ${prefix} is full (${busy_count}/${max})"
|
|
continue
|
|
fi
|
|
|
|
log "Pool ${prefix}: ${busy_count}/${max} busy, spawning up to ${available_slots} workers"
|
|
did_anything=true
|
|
|
|
# Build supervisor prompt
|
|
local prompt_text
|
|
prompt_text=$(cat <<EOF
|
|
spawn_count: ${available_slots}
|
|
exclude_work_numbers: [${exclude_numbers}]
|
|
|
|
tag prefix: "${prefix}"
|
|
Max parallel workers: ${max}
|
|
|
|
Start processing and return the list of PR/Issue numbers you launched workers for.
|
|
EOF
|
|
)
|
|
|
|
# Launch supervisor sequentially, blocking
|
|
local launched_json
|
|
launched_json=$(launch_supervisor "$supervisor_name" "BUILDER-${prefix}" "$prompt_text")
|
|
if [[ -n "$launched_json" && "$launched_json" != "[]" ]]; then
|
|
log "Launched workers for: ${launched_json}"
|
|
fi
|
|
done
|
|
|
|
# ── Step 3: Health & maintenance cycle ────────────────────────────────
|
|
if run_health_cycle; then
|
|
did_anything=true
|
|
fi
|
|
|
|
# ── Step 4: Idle sleep if nothing happened ────────────────────────────
|
|
if ! $did_anything; then
|
|
log "Nothing to do — sleeping ${SLEEP_WHEN_IDLE}s"
|
|
record_idle_event
|
|
sleep "$SLEEP_WHEN_IDLE"
|
|
fi
|
|
done
|
|
|
|
log "Loop stopped. Shutting down."
|
|
exit 0
|
|
}
|
|
|
|
main_loop
|