freemo
3d2f9ae043
The OpenCode /session/status endpoint reports all async workers as "idle" even while they are actively processing — making the previous --status busy filter always return empty and workers count as 0. Fixes across the board: - get_active_worker_tags() and count_active_workers(): new functions that filter by last_active >= (now - 2 min) instead of status field. - Pool launcher: use count_active_workers() for the busy count, and extract exclude_numbers from active sessions' tags. - Health cycle idle check: was using --status idle (returns all sessions since everything is "idle"). Now uses --status all + last_active threshold to find genuinely idle workers. - Doom loop check: was using --status busy (returns nothing). Now uses --status all + last_active < (now - 30 min) to find abandoned sessions. - fetch_list_script: guard against corrupted JSON when stderr messages from the script contaminate stdout — only accept output starting with "[" as valid JSON. - Worker launch: add --restart so stale sessions are always replaced with fresh workers. - Add IDLE_THRESHOLD_MS (3 min) constant for the health cycle.
1003 lines
36 KiB
Bash
Executable File
1003 lines
36 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# opencode-builder.sh
|
|
#
|
|
# Script-driven orchestrator for the auto-agents system.
|
|
# Queries the OpenCode server fleet, calculates worker pool gaps,
|
|
# fetches work items directly via list_prs_* / list_issues scripts,
|
|
# and launches workers via session_start.ts — no supervisor agents needed.
|
|
#
|
|
# Signal handling:
|
|
# Ctrl+C once — finishes the in-flight request, then stops the loop
|
|
# Ctrl+C twice — (within 2 seconds) kills the request and exits immediately
|
|
#
|
|
|
|
set -uo pipefail
|
|
|
|
# ── Configuration (override via environment) ──────────────────────────────
|
|
PORT="${OPENCODE_PORT:-4096}"
|
|
HOST="${OPENCODE_HOST:-127.0.0.1}"
|
|
BASE="http://${HOST}:${PORT}"
|
|
HEALTH_TIMEOUT=60
|
|
MAX_IDLE_PER_MINUTE=10
|
|
|
|
# Worker pool sizing
|
|
total_max_workers="${CA_MAX_PARALLEL_WORKERS:-4}"
|
|
# Calculate per-type max workers (same logic as auto-agents)
|
|
merge_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 7) / 8)}")
|
|
imp_max_workers="${total_max_workers}"
|
|
rev_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 1) / 2)}")
|
|
|
|
# Health check intervals (seconds)
|
|
DOOM_CHECK_INTERVAL=900 # 15 minutes
|
|
DOOM_AGE_THRESHOLD=1800 # 30 minutes
|
|
CONTINUE_BURST_WINDOW=60 # seconds
|
|
CONTINUE_BURST_LIMIT=5
|
|
SLEEP_WHEN_IDLE=30 # seconds
|
|
IDLE_THRESHOLD_MS=180000 # 3 minutes — session considered idle if last_active > this
|
|
|
|
# Script paths
|
|
SCRIPT_DIR="/app/.opencode/skills/auto-agents-system/scripts"
|
|
SESSION_LIST="npx --yes tsx ${SCRIPT_DIR}/session_list.ts"
|
|
SESSION_FIND_PREFIX="npx --yes tsx ${SCRIPT_DIR}/session_find_by_prefix.ts"
|
|
SESSION_START="npx --yes tsx ${SCRIPT_DIR}/session_start.ts"
|
|
SESSION_MESSAGES="npx --yes tsx ${SCRIPT_DIR}/session_messages.ts"
|
|
SESSION_DELETE="npx --yes tsx ${SCRIPT_DIR}/session_delete.ts"
|
|
SESSION_STOP="npx --yes tsx ${SCRIPT_DIR}/session_stop.ts"
|
|
|
|
# ── Internal state ────────────────────────────────────────────────────────
|
|
SERVER_PID=""
|
|
CURL_PID=""
|
|
STOP_LOOP=false
|
|
LAST_SIGINT=0
|
|
OWN_SERVER=false
|
|
N=0
|
|
IDLE_EVENTS_FILE=""
|
|
|
|
# In-memory tracking for health checks
|
|
declare -A QUESTION_COUNT
|
|
declare -A CONTINUE_TIMES
|
|
LAST_DOOM_CHECK=0
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────────
|
|
die() { printf "ERROR: %s\n" "$*" >&2; exit 1; }
|
|
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
|
|
|
check_opencode_running() {
|
|
if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
cleanup() {
|
|
if [[ -n "$IDLE_EVENTS_FILE" ]]; then
|
|
rm -f "$IDLE_EVENTS_FILE" 2>/dev/null || true
|
|
fi
|
|
if $OWN_SERVER && [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
|
|
log "Stopping opencode server (PID $SERVER_PID)..."
|
|
kill "$SERVER_PID" 2>/dev/null
|
|
wait "$SERVER_PID" 2>/dev/null || true
|
|
elif ! $OWN_SERVER; then
|
|
log "Leaving existing OpenCode server running (not managed by this script)"
|
|
fi
|
|
}
|
|
|
|
handle_sigint() {
|
|
local now
|
|
now=$(date +%s)
|
|
|
|
# Second Ctrl+C within 2 seconds → force-quit
|
|
if $STOP_LOOP && (( now - LAST_SIGINT <= 2 )); then
|
|
printf "\nForce quit!\n"
|
|
[[ -n "$CURL_PID" ]] && kill "$CURL_PID" 2>/dev/null || true
|
|
cleanup
|
|
exit 130
|
|
fi
|
|
|
|
# First Ctrl+C → graceful stop after current request
|
|
STOP_LOOP=true
|
|
LAST_SIGINT=$now
|
|
printf "\nWill stop after the current request completes. Ctrl+C again within 2s to force-quit.\n"
|
|
}
|
|
|
|
trap handle_sigint INT
|
|
trap cleanup EXIT
|
|
|
|
# ── Idle-event tracking ───────────────────────────────────────────────────
|
|
init_idle_tracking() {
|
|
IDLE_EVENTS_FILE="/tmp/opencode-builder-idle-events-$$.txt"
|
|
: > "$IDLE_EVENTS_FILE" 2>/dev/null || true
|
|
}
|
|
|
|
record_idle_event() {
|
|
local now
|
|
now=$(date +%s)
|
|
echo "$now" >> "$IDLE_EVENTS_FILE"
|
|
}
|
|
|
|
count_idle_events_last_minute() {
|
|
local now cutoff
|
|
now=$(date +%s)
|
|
cutoff=$((now - 60))
|
|
if [[ -f "$IDLE_EVENTS_FILE" ]]; then
|
|
awk -v c="$cutoff" '$1 > c {count++} END {print count+0}' "$IDLE_EVENTS_FILE" 2>/dev/null
|
|
else
|
|
echo 0
|
|
fi
|
|
}
|
|
|
|
# ── Session lifecycle helpers ─────────────────────────────────────────────
|
|
query_all_sessions() {
|
|
$SESSION_LIST --server "${BASE}" 2>/dev/null
|
|
}
|
|
|
|
find_sessions_by_prefix() {
|
|
local prefix="$1"
|
|
local status_filter="${2:-all}"
|
|
$SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null
|
|
}
|
|
|
|
ACTIVE_THRESHOLD_SECONDS=120
|
|
|
|
get_active_worker_tags() {
|
|
local prefix="$1"
|
|
local now_ms
|
|
now_ms=$(date +%s000)
|
|
local cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000))
|
|
find_sessions_by_prefix "$prefix" "all" | jq -r \
|
|
--arg cutoff "$cutoff_ms" \
|
|
'.[] | select(.last_active > ($cutoff_ms | tonumber)) | .tag // empty'
|
|
}
|
|
|
|
count_active_workers() {
|
|
local prefix="$1"
|
|
local now_ms
|
|
now_ms=$(date +%s000)
|
|
local cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000))
|
|
find_sessions_by_prefix "$prefix" "all" | jq \
|
|
--arg cutoff "$cutoff_ms" \
|
|
'[.[] | select(.last_active > ($cutoff_ms | tonumber))] | length'
|
|
}
|
|
|
|
create_session() {
|
|
local title="$1"
|
|
curl -sf -X POST "${BASE}/session" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"title\":\"${title}\"}" 2>/dev/null
|
|
}
|
|
|
|
delete_session() {
|
|
local sid="$1"
|
|
$SESSION_DELETE --server "${BASE}" --session-id "$sid" --force >/dev/null 2>&1 || true
|
|
}
|
|
|
|
stop_session() {
|
|
local sid="$1"
|
|
$SESSION_STOP --server "${BASE}" --session-id "$sid" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Send a message to a session and block until the LLM responds.
|
|
# Captures the assistant's response to a temp file.
|
|
send_message_capture() {
|
|
local sid="$1"
|
|
local body="$2"
|
|
local out_file="$3"
|
|
|
|
( trap '' INT
|
|
exec curl -sf -X POST "${BASE}/session/${sid}/message" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$body" \
|
|
-w "\n"
|
|
) > "$out_file" 2>/dev/null &
|
|
CURL_PID=$!
|
|
|
|
while kill -0 "$CURL_PID" 2>/dev/null; do
|
|
wait "$CURL_PID" 2>/dev/null || true
|
|
done
|
|
CURL_PID=""
|
|
}
|
|
|
|
# Poll GET /message until we see an assistant response or timeout
|
|
get_messages_poll() {
|
|
local sid="$1"
|
|
local limit="${2:-10}"
|
|
local timeout_secs="${3:-60}"
|
|
local deadline
|
|
deadline=$(($(date +%s) + timeout_secs))
|
|
|
|
while true; do
|
|
local msgs
|
|
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit "$limit" 2>/dev/null)
|
|
if [[ -n "$msgs" ]]; then
|
|
# Check if there's an assistant message
|
|
local has_assistant
|
|
has_assistant=$(echo "$msgs" | jq -r 'map(select(.info.role == "assistant")) | length')
|
|
if [[ "$has_assistant" -gt 0 ]]; then
|
|
echo "$msgs"
|
|
return 0
|
|
fi
|
|
fi
|
|
if [[ $(date +%s) -ge $deadline ]]; then
|
|
echo "[]"
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
}
|
|
|
|
# Extract the last assistant text from a messages JSON array
|
|
extract_last_assistant_text() {
|
|
local msgs="$1"
|
|
echo "$msgs" | jq -r '[.[] | select(.info.role == "assistant")] | last | if . == null then "" else (.parts // [] | map(select(.type == "text")) | .[-1].text // "") end'
|
|
}
|
|
|
|
# Extract a JSON array from text (look for [ ... ] block)
|
|
extract_json_array() {
|
|
local text="$1"
|
|
# Try to find a JSON array between [ and ]
|
|
echo "$text" | grep -oP '\[\s*[^\]]*\]' | tail -n1
|
|
}
|
|
|
|
# ── Question handling ─────────────────────────────────────────────────────
|
|
get_pending_questions() {
|
|
curl -sf "${BASE}/question" 2>/dev/null
|
|
}
|
|
|
|
answer_question() {
|
|
local qid="$1"
|
|
local text="$2"
|
|
curl -sf -X POST "${BASE}/question/${qid}/answer" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"body\":\"${text}\"}" >/dev/null 2>&1
|
|
}
|
|
|
|
reject_question() {
|
|
local qid="$1"
|
|
curl -sf -X POST "${BASE}/question/${qid}/reject" >/dev/null 2>&1
|
|
}
|
|
|
|
# ── Forgejo configuration from environment + git remote fallback ─────────────
|
|
resolve_forgejo_config() {
|
|
local url="${FORGEJO_URL:-}"
|
|
local owner="${FORGEJO_OWNER:-}"
|
|
local repo="${FORGEJO_REPO:-}"
|
|
local pat="${FORGEJO_PAT:-}"
|
|
local git_name="${GIT_USER_NAME:-}"
|
|
local git_email="${GIT_USER_EMAIL:-}"
|
|
|
|
if [[ -z "$owner" || -z "$repo" ]]; then
|
|
local remote
|
|
remote=$(git remote get-url origin 2>/dev/null)
|
|
if [[ -n "$remote" ]]; then
|
|
local proto_host
|
|
proto_host=$(echo "$remote" | sed -n 's|^\(https\?://[^/]\+\).*|\1|p')
|
|
local path_seg
|
|
path_seg=$(echo "$remote" | sed -n 's|.*://[^/]\+/\([^/]\+/[^/]\+\).*|\1|p' | sed 's/\.git$//')
|
|
local remote_owner="${path_seg%%/*}"
|
|
local remote_repo="${path_seg#*/}"
|
|
if [[ -z "$url" ]]; then
|
|
url="$proto_host"
|
|
fi
|
|
if [[ -z "$owner" ]]; then
|
|
owner="$remote_owner"
|
|
fi
|
|
if [[ -z "$repo" ]]; then
|
|
repo="$remote_repo"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
FORGEJO_URL="$url"
|
|
FORGEJO_OWNER="$owner"
|
|
FORGEJO_REPO="$repo"
|
|
FORGEJO_PAT="$pat"
|
|
GIT_USER_NAME="$git_name"
|
|
GIT_USER_EMAIL="$git_email"
|
|
}
|
|
|
|
# ── Work-group fetchers ────────────────────────────────────────────────────
|
|
fetch_list_script() {
|
|
local script_path="$1"
|
|
local forgejo_url="$2"
|
|
local forgejo_owner="$3"
|
|
local forgejo_repo="$4"
|
|
local forgejo_pat="$5"
|
|
|
|
if [[ -z "$forgejo_url" || -z "$forgejo_pat" ]]; then
|
|
echo "[]"
|
|
return
|
|
fi
|
|
|
|
local result
|
|
result=$(npx --yes tsx "$script_path" \
|
|
--url "$forgejo_url" \
|
|
--pat "$forgejo_pat" \
|
|
--owner "$forgejo_owner" \
|
|
--repo "$forgejo_repo" 2>/dev/null)
|
|
local first_char
|
|
first_char="${result%"${result#?}"}"
|
|
if [[ "$first_char" == "[" || "$first_char" == "" ]]; then
|
|
echo "$result"
|
|
else
|
|
echo "[]"
|
|
fi
|
|
}
|
|
|
|
# ── Worker launchers ───────────────────────────────────────────────────────
|
|
SESSION_START="npx --yes tsx ${SCRIPT_DIR}/session_start.ts"
|
|
|
|
# ── Worker prompt builders ─────────────────────────────────────────────────
|
|
build_impl_prompt() {
|
|
local item_json="$1"
|
|
local kind="$2"
|
|
local work_num="$3"
|
|
|
|
local title escaped_title
|
|
title=$(echo "$item_json" | jq -r '.title')
|
|
escaped_title="${title//\"/\\\"}"
|
|
|
|
local body milestone labels_json milestone_id milestone_title
|
|
if [[ "$kind" == "ISSUE" ]]; then
|
|
body=$(echo "$item_json" | jq -r '.body // ""')
|
|
milestone=$(echo "$item_json" | jq -r '.milestone')
|
|
milestone_id=$(echo "$milestone" | jq -r '.id // 0')
|
|
milestone_title=$(echo "$milestone" | jq -r '.title // ""')
|
|
labels_json=$(echo "$item_json" | jq -r '.labels')
|
|
else
|
|
body=$(echo "$item_json" | jq -r '.body // ""')
|
|
milestone=$(echo "$item_json" | jq -r '.milestone // null')
|
|
milestone_id=$(echo "$milestone" | jq -r 'if . == null then 0 else .id end')
|
|
milestone_title=$(echo "$milestone" | jq -r 'if . == null then "" else .title end')
|
|
labels_json=$(echo "$item_json" | jq -r '.labels')
|
|
fi
|
|
|
|
cat <<PROMPT
|
|
issue_number: ${work_num}
|
|
pr_number: ${work_num}
|
|
work_type: $([ "$kind" == "ISSUE" ] && echo "issue_impl" || echo "pr_fix")
|
|
issue_title: "${escaped_title}"
|
|
pr_title: "${escaped_title}"
|
|
issue_body: "${body//\"/\\\"}"
|
|
pr_body: "${body//\"/\\\"}"
|
|
milestone_id: ${milestone_id}
|
|
milestone_title: "${milestone_title}"
|
|
labels_json: ${labels_json}
|
|
forgejo_url: "${FORGEJO_URL}"
|
|
forgejo_owner: "${FORGEJO_OWNER}"
|
|
forgejo_repo: "${FORGEJO_REPO}"
|
|
forgejo_pat: "${FORGEJO_PAT}"
|
|
git_user_name: "${GIT_USER_NAME}"
|
|
git_user_email: "${GIT_USER_EMAIL}"
|
|
|
|
Implement or fix the indicated issue or pull request.
|
|
|
|
PR Compliance Checklist (MANDATORY — complete ALL items before creating a PR):
|
|
[ ] 1. CHANGELOG.md — add entry under [Unreleased] section
|
|
[ ] 2. CONTRIBUTORS.md — add or update contribution entry
|
|
[ ] 3. Commit footer — include \`ISSUES CLOSED: #<issue-number>\` in the commit message
|
|
[ ] 4. CI passes — all quality gates and tests green before requesting review
|
|
[ ] 5. BDD/Behave tests — added or updated for the changed behaviour
|
|
[ ] 6. Epic reference — PR description references the parent Epic issue number
|
|
[ ] 7. Labels — applied via forgejo-label-manager: State/In Review, Priority/<level>, MoSCoW/<level>, Type/<type>
|
|
[ ] 8. Milestone — PR assigned to the earliest open milestone matching the issue
|
|
PROMPT
|
|
}
|
|
|
|
build_merge_prompt() {
|
|
local pr_json="$1"
|
|
local pr_num="$2"
|
|
|
|
local title branch_name head_sha base_sha merge_base is_stale has_conflicts
|
|
title=$(echo "$pr_json" | jq -r '.title')
|
|
branch_name=$(echo "$pr_json" | jq -r '.head.ref')
|
|
head_sha=$(echo "$pr_json" | jq -r '.head.sha')
|
|
base_sha=$(echo "$pr_json" | jq -r '.base.sha')
|
|
merge_base=$(echo "$pr_json" | jq -r '.merge_base // empty')
|
|
if [[ "$merge_base" == "empty" || -z "$merge_base" || "$merge_base" == "null" ]]; then
|
|
merge_base="$base_sha"
|
|
fi
|
|
|
|
local stale_state mergeable
|
|
stale_state=$(echo "$pr_json" | jq -r '.stale_state // "not_stale"')
|
|
mergeable=$(echo "$pr_json" | jq -r '.mergeable')
|
|
if [[ "$stale_state" == "not_stale" ]]; then
|
|
is_stale="false"
|
|
else
|
|
is_stale="true"
|
|
fi
|
|
if [[ "$mergeable" == "false" ]]; then
|
|
has_conflicts="true"
|
|
else
|
|
has_conflicts="false"
|
|
fi
|
|
|
|
local approvals_count ci_status review_status
|
|
approvals_count=$(echo "$pr_json" | jq -r '.approvals_count // 0')
|
|
ci_status=$(echo "$pr_json" | jq -r '.ci_status // "unknown"')
|
|
if [[ "$approvals_count" -gt 0 ]]; then
|
|
review_status="approved"
|
|
else
|
|
review_status="not_approved"
|
|
fi
|
|
|
|
cat <<PROMPT
|
|
pr_number: ${pr_num}
|
|
pr_title: "${title//\"/\\\"}"
|
|
branch_name: "${branch_name}"
|
|
head_sha: "${head_sha}"
|
|
base_sha: "${base_sha}"
|
|
merge_base_sha: "${merge_base}"
|
|
is_stale: ${is_stale}
|
|
has_conflicts: ${has_conflicts}
|
|
review_status: "${review_status}"
|
|
ci_status: "${ci_status}"
|
|
approvals_count: ${approvals_count}
|
|
forgejo_url: "${FORGEJO_URL}"
|
|
forgejo_owner: "${FORGEJO_OWNER}"
|
|
forgejo_repo: "${FORGEJO_REPO}"
|
|
forgejo_pat: "${FORGEJO_PAT}"
|
|
git_user_name: "${GIT_USER_NAME}"
|
|
git_user_email: "${GIT_USER_EMAIL}"
|
|
|
|
Process the indicated Pull Request or Issue.
|
|
PROMPT
|
|
}
|
|
|
|
build_review_prompt() {
|
|
local pr_json="$1"
|
|
local pr_num="$2"
|
|
|
|
local title branch_name head_sha base_sha merge_base_sha is_stale has_conflicts
|
|
title=$(echo "$pr_json" | jq -r '.title')
|
|
branch_name=$(echo "$pr_json" | jq -r '.head.ref')
|
|
head_sha=$(echo "$pr_json" | jq -r '.head.sha')
|
|
base_sha=$(echo "$pr_json" | jq -r '.base.sha')
|
|
merge_base=$(echo "$pr_json" | jq -r '.merge_base // empty')
|
|
if [[ "$merge_base" == "empty" || -z "$merge_base" || "$merge_base" == "null" ]]; then
|
|
merge_base="$base_sha"
|
|
fi
|
|
merge_base_sha="$merge_base"
|
|
|
|
local stale_state mergeable
|
|
stale_state=$(echo "$pr_json" | jq -r '.stale_state // "not_stale"')
|
|
mergeable=$(echo "$pr_json" | jq -r '.mergeable')
|
|
if [[ "$stale_state" == "not_stale" ]]; then
|
|
is_stale="false"
|
|
else
|
|
is_stale="true"
|
|
fi
|
|
if [[ "$mergeable" == "false" ]]; then
|
|
has_conflicts="true"
|
|
else
|
|
has_conflicts="false"
|
|
fi
|
|
|
|
local approvals_count ci_status
|
|
approvals_count=$(echo "$pr_json" | jq -r '.approvals_count // 0')
|
|
ci_status=$(echo "$pr_json" | jq -r '.ci_status // "unknown"')
|
|
|
|
cat <<PROMPT
|
|
pr_number: ${pr_num}
|
|
pr_title: "${title//\"/\\\"}"
|
|
branch_name: "${branch_name}"
|
|
head_sha: "${head_sha}"
|
|
base_sha: "${base_sha}"
|
|
merge_base_sha: "${merge_base_sha}"
|
|
is_stale: ${is_stale}
|
|
has_conflicts: ${has_conflicts}
|
|
approvals_count: ${approvals_count}
|
|
ci_status: "${ci_status}"
|
|
forgejo_url: "${FORGEJO_URL}"
|
|
forgejo_owner: "${FORGEJO_OWNER}"
|
|
forgejo_repo: "${FORGEJO_REPO}"
|
|
forgejo_pat: "${FORGEJO_PAT}"
|
|
git_user_name: "${GIT_USER_NAME}"
|
|
git_user_email: "${GIT_USER_EMAIL}"
|
|
|
|
Process the indicated Pull Request or Issue.
|
|
PROMPT
|
|
}
|
|
|
|
# ── Fleet analysis ────────────────────────────────────────────────────────
|
|
# Returns the list of busy worker tags for a given prefix, one per line.
|
|
get_busy_worker_tags() {
|
|
local prefix="$1"
|
|
find_sessions_by_prefix "$prefix" "busy" | jq -r '.[].tag // empty'
|
|
}
|
|
|
|
# Extract the PR/Issue number from a worker tag.
|
|
# Tags look like: AUTO-IMP-PR-42, AUTO-IMP-ISSUE-42, AUTO-MRG-PR-42, AUTO-REV-PR-42
|
|
extract_work_number() {
|
|
local tag="$1"
|
|
echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p'
|
|
}
|
|
|
|
# ── Duplicate worker guard ────────────────────────────────────────────────
|
|
deduplicate_workers() {
|
|
local prefix="$1"
|
|
local workers_json
|
|
workers_json=$(find_sessions_by_prefix "$prefix" "busy")
|
|
# Group by tag, keep only the most recently active for each tag
|
|
local duplicates
|
|
duplicates=$(echo "$workers_json" | jq -r '
|
|
group_by(.tag)
|
|
| map(select(length > 1))
|
|
| map(sort_by(.last_active) | reverse | .[1:])
|
|
| flatten
|
|
| .[].id
|
|
')
|
|
if [[ -n "$duplicates" ]]; then
|
|
while IFS= read -r dup_id; do
|
|
[[ -n "$dup_id" ]] || continue
|
|
log "Duplicate worker guard: deleting older session ${dup_id}"
|
|
delete_session "$dup_id"
|
|
done <<< "$duplicates"
|
|
fi
|
|
}
|
|
|
|
# ── Disk cleanup ──────────────────────────────────────────────────────────
|
|
cleanup_orphan_tmp_dirs() {
|
|
local prefix="$1"
|
|
local active_tags
|
|
active_tags=$(find_sessions_by_prefix "$prefix" "all" | jq -r '.[].tag // empty' | sort -u)
|
|
|
|
# Look for /tmp directories that match the agent pattern for this prefix
|
|
# Expected pattern: /tmp/<agent_name>-<identifier>-<timestamp>/
|
|
# We can't know the exact agent name, so we look for directories with numbers
|
|
# that might be identifiers, and cross-check against active tags.
|
|
local tag_pattern="${prefix}"
|
|
while IFS= read -r -d '' dir; do
|
|
[[ -d "$dir" ]] || continue
|
|
# Extract any numeric identifier from the directory basename
|
|
local dir_id
|
|
dir_id=$(basename "$dir" | grep -oP '\d+' | tail -n1)
|
|
if [[ -n "$dir_id" ]]; then
|
|
# Check if any active tag contains this number
|
|
local is_active=false
|
|
while IFS= read -r atag; do
|
|
[[ -n "$atag" ]] || continue
|
|
if [[ "$atag" == *"${dir_id}"* ]]; then
|
|
is_active=true
|
|
break
|
|
fi
|
|
done <<< "$active_tags"
|
|
if ! $is_active; then
|
|
log "Disk cleanup: removing orphan ${dir}"
|
|
rm -rf "$dir"
|
|
fi
|
|
fi
|
|
done < <(find /tmp -maxdepth 1 -type d -name "*${tag_pattern}*" -print0 2>/dev/null)
|
|
}
|
|
|
|
# ── Health checks ─────────────────────────────────────────────────────────
|
|
run_health_cycle() {
|
|
local now
|
|
now=$(date +%s)
|
|
local did_something=false
|
|
|
|
# 1) Question handling
|
|
local questions_json
|
|
questions_json=$(get_pending_questions)
|
|
if [[ -n "$questions_json" && "$questions_json" != "[]" && "$questions_json" != "null" ]]; then
|
|
local qids sids
|
|
qids=$(echo "$questions_json" | jq -r '.[].id')
|
|
sids=$(echo "$questions_json" | jq -r '.[].sessionID')
|
|
while IFS= read -r qid && IFS= read -r sid <&3; do
|
|
[[ -n "$qid" ]] || continue
|
|
[[ -n "$sid" ]] || continue
|
|
local cnt="${QUESTION_COUNT[$sid]:-0}"
|
|
if [[ "$cnt" -eq 0 ]]; then
|
|
log "Question from ${sid} (first time) — answering with anti-question text"
|
|
answer_question "$qid" "you are never under any circumstances to ask questions or pass along questions from subagents, use your best judgement and do not ask questions again"
|
|
QUESTION_COUNT[$sid]=1
|
|
else
|
|
log "Question from ${sid} (repeat) — deleting session"
|
|
delete_session "$sid"
|
|
unset QUESTION_COUNT[$sid]
|
|
fi
|
|
did_something=true
|
|
done <<< "$qids" 3<<< "$sids"
|
|
fi
|
|
|
|
# 2) Idle worker check — use last_active threshold to find genuinely idle sessions
|
|
local all_prefixes=("AUTO-IMP" "AUTO-MRG" "AUTO-REV")
|
|
for prefix in "${all_prefixes[@]}"; do
|
|
local idle_workers
|
|
idle_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \
|
|
--arg cutoff "$(($(date +%s) * 1000 - IDLE_THRESHOLD_MS))" \
|
|
'.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty')
|
|
while IFS= read -r sid; do
|
|
[[ -n "$sid" ]] || continue
|
|
local msgs
|
|
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit 10 2>/dev/null)
|
|
if [[ -z "$msgs" || "$msgs" == "[]" ]]; then
|
|
log "Idle worker ${sid} has no messages — deleting"
|
|
delete_session "$sid"
|
|
did_something=true
|
|
continue
|
|
fi
|
|
# Check if it appears partially done (last assistant message suggests more work)
|
|
local last_text
|
|
last_text=$(extract_last_assistant_text "$msgs")
|
|
# Heuristic: if the last text contains completion signals, delete it
|
|
if echo "$last_text" | grep -qiE '(done|finished|complete|exiting|all tests pass|committed|merged|review submitted)'; then
|
|
log "Idle worker ${sid} appears finished — deleting"
|
|
delete_session "$sid"
|
|
did_something=true
|
|
continue
|
|
fi
|
|
# Otherwise, send continue and track
|
|
local now_ms
|
|
now_ms=$(date +%s)
|
|
local times_str="${CONTINUE_TIMES[$sid]:-}"
|
|
local cutoff=$((now_ms - CONTINUE_BURST_WINDOW))
|
|
# Store as space-delimited timestamps to avoid newline parsing issues
|
|
local new_times_arr=()
|
|
local count=0
|
|
if [[ -n "$times_str" ]]; then
|
|
for ts in $times_str; do
|
|
[[ -n "$ts" ]] || continue
|
|
if (( ts > cutoff )); then
|
|
new_times_arr+=("$ts")
|
|
((count++))
|
|
fi
|
|
done
|
|
fi
|
|
new_times_arr+=("$now_ms")
|
|
((count++))
|
|
CONTINUE_TIMES[$sid]=$(printf '%s ' "${new_times_arr[@]}")
|
|
if [[ "$count" -gt $CONTINUE_BURST_LIMIT ]]; then
|
|
log "Idle worker ${sid} received ${count} continues in ${CONTINUE_BURST_WINDOW}s — deleting"
|
|
delete_session "$sid"
|
|
unset CONTINUE_TIMES[$sid]
|
|
did_something=true
|
|
else
|
|
log "Idle worker ${sid} — sending continue"
|
|
local cont_body
|
|
cont_body=$(jq -nc '{agent: "", parts: [{type: "text", text: "continue"}]}')
|
|
send_message_capture "$sid" "$cont_body" "/tmp/continue-${sid}.txt"
|
|
did_something=true
|
|
fi
|
|
done <<< "$idle_workers"
|
|
done
|
|
|
|
# 3) Doom-loop check (every 15 minutes) — active sessions with no recent tool calls
|
|
if [[ $((now - LAST_DOOM_CHECK)) -ge $DOOM_CHECK_INTERVAL ]]; then
|
|
LAST_DOOM_CHECK=$now
|
|
for prefix in "${all_prefixes[@]}"; do
|
|
local busy_workers
|
|
busy_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \
|
|
--arg cutoff "$(( (now - DOOM_AGE_THRESHOLD) * 1000 ))" \
|
|
'.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty')
|
|
while IFS= read -r sid; do
|
|
[[ -n "$sid" ]] || continue
|
|
log "Doom-loop check for ${sid} (busy >30 min)"
|
|
local eval_tag="BUILDER-HEALTH-${sid: -6}"
|
|
local eval_title="[${eval_tag}] worker-health-evaluator"
|
|
local eval_prompt="session_id: ${sid}
|
|
|
|
Evaluate the health of this session."
|
|
local eval_session_json
|
|
eval_session_json=$(create_session "$eval_title")
|
|
local eval_sid
|
|
eval_sid=$(echo "$eval_session_json" | jq -r '.id')
|
|
if [[ -n "$eval_sid" && "$eval_sid" != "null" ]]; then
|
|
local eval_tmp="/tmp/builder-prompt-${eval_tag}-$$.txt"
|
|
printf '%s' "$eval_prompt" > "$eval_tmp"
|
|
local eval_body
|
|
eval_body=$(jq -nc --arg a "worker-health-evaluator" --rawfile t "$eval_tmp" '{agent: $a, parts: [{type: "text", text: $t}]}')
|
|
curl -sf -X POST "${BASE}/session/${eval_sid}/prompt_async" -H "Content-Type: application/json" -d "$eval_body" >/dev/null 2>&1
|
|
local eval_deadline
|
|
eval_deadline=$(($(date +%s) + 120))
|
|
while [[ $(date +%s) -lt $eval_deadline ]]; do
|
|
local eval_status
|
|
eval_status=$(curl -sf "${BASE}/session/status" 2>/dev/null | jq -r --arg sid "$eval_sid" '.[$sid].type // "idle"')
|
|
[[ "$eval_status" == "idle" ]] && break
|
|
sleep 2
|
|
done
|
|
local eval_msgs
|
|
eval_msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$eval_sid" --limit 10 2>/dev/null)
|
|
local eval_text
|
|
eval_text=$(extract_last_assistant_text "$eval_msgs")
|
|
delete_session "$eval_sid"
|
|
rm -f "$eval_tmp"
|
|
local is_unrecoverable
|
|
is_unrecoverable=$(echo "$eval_text" | jq -r '.unrecoverable // false' 2>/dev/null || echo "false")
|
|
if [[ "$is_unrecoverable" == "true" ]]; then
|
|
log "Session ${sid} is unrecoverable — deleting"
|
|
delete_session "$sid"
|
|
did_something=true
|
|
fi
|
|
fi
|
|
done <<< "$busy_workers"
|
|
done
|
|
fi
|
|
|
|
if $did_something; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ── Preflight checks ─────────────────────────────────────────────────────
|
|
for cmd in curl jq opencode npx; do
|
|
command -v "$cmd" &>/dev/null || die "'$cmd' is required but not found in PATH"
|
|
done
|
|
|
|
# ── Start the opencode server ─────────────────────────────────────────────
|
|
if check_opencode_running; then
|
|
echo "┌─────────────────────────────────────────────────────────────────┐"
|
|
echo "│ ⚠️ WARNING: OpenCode server already running on ${HOST}:${PORT}"
|
|
echo "│ 🔗 Connecting to existing server instead of starting new one"
|
|
echo "│ 💡 The script will NOT manage this server's lifecycle"
|
|
echo "└─────────────────────────────────────────────────────────────────┘"
|
|
echo ""
|
|
log "Using existing OpenCode server at ${HOST}:${PORT}"
|
|
OWN_SERVER=false
|
|
else
|
|
log "Starting opencode server on ${HOST}:${PORT}..."
|
|
opencode serve --port "$PORT" --hostname "$HOST" &
|
|
SERVER_PID=$!
|
|
OWN_SERVER=true
|
|
|
|
log "Waiting for server to become healthy (up to ${HEALTH_TIMEOUT}s)..."
|
|
healthy=false
|
|
for (( i = 1; i <= HEALTH_TIMEOUT; i++ )); do
|
|
if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then
|
|
healthy=true
|
|
break
|
|
fi
|
|
kill -0 "$SERVER_PID" 2>/dev/null || die "Server process exited unexpectedly"
|
|
sleep 1
|
|
done
|
|
$healthy || die "Server did not become healthy within ${HEALTH_TIMEOUT}s"
|
|
log "Server is ready."
|
|
fi
|
|
|
|
# ── Main orchestration loop ───────────────────────────────────────────────
|
|
main_loop() {
|
|
log "Starting orchestration loop."
|
|
log "Worker pools: merge=${merge_max_workers} imp=${imp_max_workers} rev=${rev_max_workers}"
|
|
|
|
while ! $STOP_LOOP; do
|
|
N=$((N + 1))
|
|
|
|
# Check idle-event rate in the last minute
|
|
idle_count=$(count_idle_events_last_minute)
|
|
if (( idle_count > MAX_IDLE_PER_MINUTE )); then
|
|
log "── iteration #${N} ── TOO MANY IDLE EVENTS (${idle_count}) — skipping cycle"
|
|
sleep 5
|
|
continue
|
|
fi
|
|
|
|
log "── iteration #${N} ──"
|
|
local did_anything=false
|
|
|
|
# ── Step 1: Query fleet ───────────────────────────────────────────────
|
|
local all_sessions_json
|
|
all_sessions_json=$(query_all_sessions)
|
|
if [[ -z "$all_sessions_json" ]]; then
|
|
log "No sessions found."
|
|
fi
|
|
|
|
# ── Step 2: Per-type analysis & direct worker launch ──────────────────
|
|
resolve_forgejo_config
|
|
|
|
if [[ -z "$FORGEJO_PAT" ]]; then
|
|
log "FORGEJO_PAT not set — skipping worker launch"
|
|
else
|
|
# Work-group definitions: label|scripts|kind
|
|
# kind: PR / ISSUE / AUTO (auto-detect from script output)
|
|
declare -a IMP_WORK_GROUPS=(
|
|
"CI Failing PRs|${SCRIPT_DIR}/list_prs_ci_failing.ts|PR"
|
|
"Addressed Changes CI Passing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_passing.ts|PR"
|
|
"Addressed Changes CI Failing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_failing.ts|PR"
|
|
"Changes Requested|${SCRIPT_DIR}/list_prs_changes_requested.ts|PR"
|
|
"CI Missing|${SCRIPT_DIR}/list_prs_missing_ci_checks.ts|PR"
|
|
"No Active Review CI Passing|${SCRIPT_DIR}/list_prs_no_active_review_ci_passing.ts|PR"
|
|
"No Active Review CI Failing|${SCRIPT_DIR}/list_prs_no_active_review_ci_failing.ts|PR"
|
|
"Needs Review Not Stale|${SCRIPT_DIR}/list_prs_needs_review_not_stale.ts|PR"
|
|
"Needs Review Stale Clean|${SCRIPT_DIR}/list_prs_needs_review_stale_clean.ts|PR"
|
|
"Needs Review Stale Conflicts|${SCRIPT_DIR}/list_prs_needs_review_stale_conflicts.ts|PR"
|
|
"Open Issues|${SCRIPT_DIR}/list_issues.ts|ISSUE"
|
|
)
|
|
declare -a MRG_WORK_GROUPS=(
|
|
"Ready to Merge|${SCRIPT_DIR}/list_prs_ready_to_merge.ts|PR"
|
|
"Stale No Conflicts|${SCRIPT_DIR}/list_prs_stale_clean.ts|PR"
|
|
"Stale Has Conflicts|${SCRIPT_DIR}/list_prs_stale_conflicts.ts|PR"
|
|
)
|
|
declare -a REV_WORK_GROUPS=(
|
|
"No Active Review CI Passing|${SCRIPT_DIR}/list_prs_no_active_review_ci_passing.ts|PR"
|
|
"No Active Review CI Failing|${SCRIPT_DIR}/list_prs_no_active_review_ci_failing.ts|PR"
|
|
"Needs Review Not Stale|${SCRIPT_DIR}/list_prs_needs_review_not_stale.ts|PR"
|
|
"Needs Review Stale Clean|${SCRIPT_DIR}/list_prs_needs_review_stale_clean.ts|PR"
|
|
"Needs Review Stale Conflicts|${SCRIPT_DIR}/list_prs_needs_review_stale_conflicts.ts|PR"
|
|
"Addressed Changes CI Passing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_passing.ts|PR"
|
|
"Addressed Changes CI Failing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_failing.ts|PR"
|
|
"Changes Requested|${SCRIPT_DIR}/list_prs_changes_requested.ts|PR"
|
|
"CI Failing|${SCRIPT_DIR}/list_prs_ci_failing.ts|PR"
|
|
)
|
|
|
|
# Pool definitions: prefix | agent | max | work-groups-array-name
|
|
declare -a pools=(
|
|
"AUTO-IMP|implementation-worker|${imp_max_workers}|IMP_WORK_GROUPS"
|
|
"AUTO-MRG|pr-merge-worker|${merge_max_workers}|MRG_WORK_GROUPS"
|
|
"AUTO-REV|pr-review-worker|${rev_max_workers}|REV_WORK_GROUPS"
|
|
)
|
|
|
|
for pool_spec in "${pools[@]}"; do
|
|
IFS='|' read -r prefix agent max wg_array_name <<< "$pool_spec"
|
|
|
|
deduplicate_workers "$prefix"
|
|
cleanup_orphan_tmp_dirs "$prefix"
|
|
|
|
local exclude_numbers=","
|
|
local all_tags_json
|
|
all_tags_json=$(find_sessions_by_prefix "$prefix" "all")
|
|
local now_ms cutoff_ms
|
|
now_ms=$(date +%s000)
|
|
cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000))
|
|
local active_tags_json
|
|
active_tags_json=$(echo "$all_tags_json" | jq --arg cutoff "$cutoff_ms" \
|
|
'[.[] | select(.last_active > ($cutoff_ms | tonumber))]')
|
|
local busy_count
|
|
busy_count=$(echo "$active_tags_json" | jq 'length')
|
|
while IFS= read -r tag; do
|
|
[[ -n "$tag" ]] || continue
|
|
local num
|
|
num=$(echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p')
|
|
if [[ -n "$num" ]]; then
|
|
exclude_numbers="${exclude_numbers}${num},"
|
|
fi
|
|
done < <(echo "$active_tags_json" | jq -r '.[].tag // empty')
|
|
|
|
log "Pool ${prefix}: ${busy_count} active (${max} max) — excluded: $(echo "$active_tags_json" | jq 'length') recent"
|
|
|
|
local available_slots=$((max - busy_count))
|
|
if [[ "$available_slots" -le 0 ]]; then
|
|
log "Pool ${prefix} is full (${busy_count}/${max})"
|
|
continue
|
|
fi
|
|
|
|
log "Pool ${prefix}: ${busy_count}/${max} busy, launching up to ${available_slots} workers directly"
|
|
did_anything=true
|
|
|
|
local wg_array_ref="${wg_array_name}[@]"
|
|
local work_groups=("${!wg_array_ref}")
|
|
|
|
local launched=0
|
|
for type_spec in "${work_groups[@]}"; do
|
|
IFS='|' read -r wg_label wg_scripts wg_kind <<< "$type_spec"
|
|
|
|
local scripts_arr
|
|
IFS=',' read -ra scripts_arr <<< "$wg_scripts"
|
|
for script_path in "${scripts_arr[@]}"; do
|
|
[[ -n "$script_path" ]] || continue
|
|
|
|
local json
|
|
json=$(fetch_list_script "$script_path" "$FORGEJO_URL" "$FORGEJO_OWNER" "$FORGEJO_REPO" "$FORGEJO_PAT")
|
|
local count
|
|
count=$(echo "$json" | jq 'length' 2>/dev/null || echo 0)
|
|
if [[ "$count" -eq 0 || "$count" == "null" ]]; then
|
|
continue
|
|
fi
|
|
|
|
local -a items_to_launch=()
|
|
while IFS= read -r item; do
|
|
[[ -n "$item" ]] || continue
|
|
|
|
local work_num
|
|
work_num=$(echo "$item" | jq -r '.number')
|
|
if [[ -z "$work_num" || "$work_num" == "null" ]]; then
|
|
continue
|
|
fi
|
|
if echo ",${exclude_numbers}," | grep -q ",${work_num},"; then
|
|
continue
|
|
fi
|
|
|
|
local kind="$wg_kind"
|
|
if [[ "$kind" == "AUTO" ]]; then
|
|
if echo "$item" | jq -e '.head // empty' >/dev/null 2>&1; then
|
|
kind="PR"
|
|
else
|
|
kind="ISSUE"
|
|
fi
|
|
fi
|
|
|
|
items_to_launch+=("${item}|${work_num}|${kind}")
|
|
done < <(echo "$json" | jq -c '.[]')
|
|
|
|
if [[ ${#items_to_launch[@]} -eq 0 ]]; then
|
|
continue
|
|
fi
|
|
|
|
local actual_launch_count=0
|
|
[[ ${#items_to_launch[@]} -lt $((available_slots - launched)) ]] \
|
|
&& actual_launch_count=${#items_to_launch[@]} \
|
|
|| actual_launch_count=$((available_slots - launched))
|
|
|
|
log "Pool ${prefix}: ${#items_to_launch[@]} items in '${wg_label}', launching up to ${actual_launch_count} in parallel"
|
|
|
|
local -a pids=()
|
|
local -a tmp_files=()
|
|
local -a work_nums=()
|
|
|
|
for ((i = 0; i < actual_launch_count; i++)); do
|
|
local item_with_meta="${items_to_launch[i]}"
|
|
local item="${item_with_meta%%|*}"
|
|
local rest="${item_with_meta#*|}"
|
|
local work_num="${rest%%|*}"
|
|
local kind="${rest##*|}"
|
|
|
|
local worker_tag="${prefix}-${kind}-${work_num}"
|
|
local tmp_prompt="/tmp/worker-prompt-${worker_tag// /-}-$$-${i}.txt"
|
|
|
|
local prompt_text
|
|
case "$agent" in
|
|
*implementation*)
|
|
prompt_text=$(build_impl_prompt "$item" "$kind" "$work_num")
|
|
;;
|
|
*merge*)
|
|
prompt_text=$(build_merge_prompt "$item" "$work_num")
|
|
;;
|
|
*review*)
|
|
prompt_text=$(build_review_prompt "$item" "$work_num")
|
|
;;
|
|
esac
|
|
|
|
printf '%s' "$prompt_text" > "$tmp_prompt"
|
|
|
|
$SESSION_START --server "${BASE}" --tag "$worker_tag" --agent "$agent" --prompt-file "$tmp_prompt" --restart \
|
|
>/dev/null 2>&1 &
|
|
pids+=($!)
|
|
tmp_files+=("$tmp_prompt")
|
|
work_nums+=("$work_num")
|
|
done
|
|
|
|
for ((i = 0; i < ${#pids[@]}; i++)); do
|
|
if wait "${pids[i]}" 2>/dev/null; then
|
|
exclude_numbers="${exclude_numbers}${work_nums[i]},"
|
|
((launched++))
|
|
did_anything=true
|
|
log "[${prefix}] launched worker ${work_nums[i]} (${agent})"
|
|
else
|
|
log "[${prefix}] FAILED worker ${work_nums[i]}"
|
|
fi
|
|
rm -f "${tmp_files[i]}"
|
|
done
|
|
|
|
if (( launched > 0 )); then
|
|
log "Pool ${prefix}: launched ${launched} from '${wg_label}'"
|
|
break 3
|
|
fi
|
|
done
|
|
done
|
|
|
|
if (( launched == 0 )); then
|
|
log "Pool ${prefix}: no available work items found"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# ── Step 3: Health & maintenance cycle ────────────────────────────────
|
|
if run_health_cycle; then
|
|
did_anything=true
|
|
fi
|
|
|
|
# ── Step 4: Idle sleep if nothing happened ────────────────────────────
|
|
if ! $did_anything; then
|
|
log "Nothing to do — sleeping ${SLEEP_WHEN_IDLE}s"
|
|
record_idle_event
|
|
sleep "$SLEEP_WHEN_IDLE"
|
|
fi
|
|
done
|
|
|
|
log "Loop stopped. Shutting down."
|
|
exit 0
|
|
}
|
|
|
|
main_loop
|