freemo
d4f0660aa2
opencode-builder.sh: major reliability and correctness fixes: - Parallel pre-fetch per pool: ALL list_prs_*.ts scripts for a pool now launch in parallel (background jobs), results cached to temp files. Each pool iteration fetches all work-group data in ONE parallel batch (~90s) instead of 11 sequential ~90s calls (~990s). The bottleneck is Forgejo API speed, not sequential script execution. - Timeouts everywhere: SCRIPT_TIMEOUT_SEC raised 60s→300s (Forgejo is slow). find_sessions_by_prefix and other session_* scripts get 30s timeout via the timeout(1) utility. fetch_list_script catches timeout exit codes (124, 137, 139, 143) and returns [] gracefully. - Prompt builders fixed: build_impl_prompt, build_merge_prompt, build_review_prompt now collapse newlines in title/body via tr+sed so the prompt file stays valid as plain text. Full item JSON embedded as compact single-line (jq -c .) at end of prompt. - Verbose step logging: [step2], [pool:NAME] prefixes throughout main loop show exactly which stage is running and where any hang occurs. - BUILDER_DEBUG=1 env var enables bash -x tracing for deep debugging. Note: find_sessions_by_prefix --status busy returns empty (OpenCode /session/status only reports the supervisor session). busy_count=0 when all session status fields are empty, making all slots available. The total session count is logged alongside active count for visibility.
994 lines
38 KiB
Bash
Executable File
994 lines
38 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
#
|
||
# opencode-builder.sh
|
||
#
|
||
# Script-driven orchestrator for the auto-agents system.
|
||
# Queries the OpenCode server fleet, calculates worker pool gaps,
|
||
# fetches work items directly via list_prs_* / list_issues scripts,
|
||
# and launches workers via session_start.ts — no supervisor agents needed.
|
||
|
||
set -uo pipefail
|
||
|
||
# ── Debug mode ─────────────────────────────────────────────────────────────
|
||
if [[ "${BUILDER_DEBUG:-0}" == "1" ]]; then
|
||
set -x
|
||
log "DEBUG MODE ENABLED"
|
||
fi
|
||
|
||
# ── Configuration (override via environment) ──────────────────────────────
|
||
PORT="${OPENCODE_PORT:-4096}"
|
||
HOST="${OPENCODE_HOST:-127.0.0.1}"
|
||
BASE="http://${HOST}:${PORT}"
|
||
HEALTH_TIMEOUT=60
|
||
MAX_IDLE_PER_MINUTE=10
|
||
SCRIPT_TIMEOUT_SEC=300 # timeout for list_prs / list_issues scripts (Forgejo is slow — allow 5 min)
|
||
SESSION_SCRIPT_TIMEOUT=30 # timeout for session_* scripts
|
||
|
||
# Worker pool sizing
|
||
total_max_workers="${CA_MAX_PARALLEL_WORKERS:-4}"
|
||
# Calculate per-type max workers (same logic as auto-agents)
|
||
merge_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 7) / 8)}")
|
||
imp_max_workers="${total_max_workers}"
|
||
rev_max_workers=$(awk "BEGIN {print int((${total_max_workers} + 1) / 2)}")
|
||
|
||
# Health check intervals (seconds)
|
||
DOOM_CHECK_INTERVAL=900 # 15 minutes
|
||
DOOM_AGE_THRESHOLD=1800 # 30 minutes
|
||
CONTINUE_BURST_WINDOW=60 # seconds
|
||
CONTINUE_BURST_LIMIT=5
|
||
SLEEP_WHEN_IDLE=30 # seconds
|
||
IDLE_THRESHOLD_MS=180000 # 3 minutes — session considered idle if last_active older than this
|
||
|
||
# Script paths
|
||
SCRIPT_DIR="/app/.opencode/skills/auto-agents-system/scripts"
|
||
SESSION_LIST="npx --yes tsx ${SCRIPT_DIR}/session_list.ts"
|
||
SESSION_FIND_PREFIX="npx --yes tsx ${SCRIPT_DIR}/session_find_by_prefix.ts"
|
||
SESSION_START="npx --yes tsx ${SCRIPT_DIR}/session_start.ts"
|
||
SESSION_MESSAGES="npx --yes tsx ${SCRIPT_DIR}/session_messages.ts"
|
||
SESSION_DELETE="npx --yes tsx ${SCRIPT_DIR}/session_delete.ts"
|
||
SESSION_STOP="npx --yes tsx ${SCRIPT_DIR}/session_stop.ts"
|
||
|
||
# ── Internal state ───────────────────────────────────────────────────────
|
||
SERVER_PID=""
|
||
CURL_PID=""
|
||
STOP_LOOP=false
|
||
LAST_SIGINT=0
|
||
OWN_SERVER=false
|
||
N=0
|
||
IDLE_EVENTS_FILE=""
|
||
|
||
# In-memory tracking for health checks
|
||
declare -A QUESTION_COUNT
|
||
declare -A CONTINUE_TIMES
|
||
LAST_DOOM_CHECK=0
|
||
|
||
# ── Helpers ───────────────────────────────────────────────────────────────
|
||
die() { printf "ERROR: %s\n" "$*" >&2; exit 1; }
|
||
log() { printf "[%s] %s\n" "$(date +%H:%M:%S)" "$*"; }
|
||
|
||
check_opencode_running() {
|
||
if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then
|
||
return 0
|
||
else
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
cleanup() {
|
||
if [[ -n "$IDLE_EVENTS_FILE" ]]; then
|
||
rm -f "$IDLE_EVENTS_FILE" 2>/dev/null || true
|
||
fi
|
||
if $OWN_SERVER && [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
|
||
log "Stopping opencode server (PID $SERVER_PID)..."
|
||
kill "$SERVER_PID" 2>/dev/null
|
||
wait "$SERVER_PID" 2>/dev/null || true
|
||
elif ! $OWN_SERVER; then
|
||
log "Leaving existing OpenCode server running (not managed by this script)"
|
||
fi
|
||
}
|
||
|
||
handle_sigint() {
|
||
local now
|
||
now=$(date +%s)
|
||
|
||
if $STOP_LOOP && (( now - LAST_SIGINT <= 2 )); then
|
||
printf "\nForce quit!\n"
|
||
[[ -n "$CURL_PID" ]] && kill "$CURL_PID" 2>/dev/null || true
|
||
cleanup
|
||
exit 130
|
||
fi
|
||
|
||
STOP_LOOP=true
|
||
LAST_SIGINT=$now
|
||
printf "\nWill stop after the current request completes. Ctrl+C again within 2s to force-quit.\n"
|
||
}
|
||
|
||
trap handle_sigint INT
|
||
trap cleanup EXIT
|
||
|
||
# ── Idle-event tracking ───────────────────────────────────────────────────
|
||
init_idle_tracking() {
|
||
IDLE_EVENTS_FILE="/tmp/opencode-builder-idle-events-$$.txt"
|
||
: > "$IDLE_EVENTS_FILE" 2>/dev/null || true
|
||
}
|
||
|
||
record_idle_event() {
|
||
local now
|
||
now=$(date +%s)
|
||
echo "$now" >> "$IDLE_EVENTS_FILE"
|
||
}
|
||
|
||
count_idle_events_last_minute() {
|
||
local now cutoff
|
||
now=$(date +%s)
|
||
cutoff=$((now - 60))
|
||
if [[ -f "$IDLE_EVENTS_FILE" ]]; then
|
||
awk -v c="$cutoff" '$1 > c {count++} END {print count+0}' "$IDLE_EVENTS_FILE" 2>/dev/null
|
||
else
|
||
echo 0
|
||
fi
|
||
}
|
||
|
||
# ── Session lifecycle helpers ─────────────────────────────────────────────
|
||
query_all_sessions() {
|
||
$SESSION_LIST --server "${BASE}" 2>/dev/null
|
||
}
|
||
|
||
find_sessions_by_prefix() {
|
||
local prefix="$1"
|
||
local status_filter="${2:-all}"
|
||
timeout "$SESSION_SCRIPT_TIMEOUT" $SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null
|
||
}
|
||
|
||
get_busy_worker_tags() {
|
||
local prefix="$1"
|
||
local now_ms
|
||
now_ms=$(date +%s000)
|
||
local cutoff_ms=$((now_ms - 120000))
|
||
find_sessions_by_prefix "$prefix" "all" | jq -r \
|
||
--arg cutoff "$cutoff_ms" \
|
||
'.[] | select(.last_active >= ($cutoff | tonumber)) | .tag // empty'
|
||
}
|
||
|
||
create_session() {
|
||
local title="$1"
|
||
curl -sf -X POST "${BASE}/session" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"title\":\"${title}\"}" 2>/dev/null
|
||
}
|
||
|
||
delete_session() {
|
||
local sid="$1"
|
||
$SESSION_DELETE --server "${BASE}" --session-id "$sid" --force >/dev/null 2>&1 || true
|
||
}
|
||
|
||
stop_session() {
|
||
local sid="$1"
|
||
$SESSION_STOP --server "${BASE}" --session-id "$sid" >/dev/null 2>&1 || true
|
||
}
|
||
|
||
send_message_capture() {
|
||
local sid="$1"
|
||
local body="$2"
|
||
local out_file="$3"
|
||
|
||
( trap '' INT
|
||
exec curl -sf -X POST "${BASE}/session/${sid}/message" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$body" \
|
||
-w "\n"
|
||
) > "$out_file" 2>/dev/null &
|
||
CURL_PID=$!
|
||
|
||
while kill -0 "$CURL_PID" 2>/dev/null; do
|
||
wait "$CURL_PID" 2>/dev/null || true
|
||
done
|
||
CURL_PID=""
|
||
}
|
||
|
||
get_messages_poll() {
|
||
local sid="$1"
|
||
local limit="${2:-10}"
|
||
local timeout_secs="${3:-60}"
|
||
local deadline
|
||
deadline=$(($(date +%s) + timeout_secs))
|
||
|
||
while true; do
|
||
local msgs
|
||
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit "$limit" 2>/dev/null)
|
||
if [[ -n "$msgs" ]]; then
|
||
local has_assistant
|
||
has_assistant=$(echo "$msgs" | jq -r 'map(select(.info.role == "assistant")) | length')
|
||
if [[ "$has_assistant" -gt 0 ]]; then
|
||
echo "$msgs"
|
||
return 0
|
||
fi
|
||
fi
|
||
if [[ $(date +%s) -ge $deadline ]]; then
|
||
echo "[]"
|
||
return 1
|
||
fi
|
||
sleep 2
|
||
done
|
||
}
|
||
|
||
extract_last_assistant_text() {
|
||
local msgs="$1"
|
||
echo "$msgs" | jq -r '[.[] | select(.info.role == "assistant")] | last | if . == null then "" else (.parts // [] | map(select(.type == "text")) | .[-1].text // "") end'
|
||
}
|
||
|
||
extract_json_array() {
|
||
local text="$1"
|
||
echo "$text" | grep -oP '\[\s*[^\]]*\]' | tail -n1
|
||
}
|
||
|
||
# ── Question handling ─────────────────────────────────────────────────────
|
||
get_pending_questions() {
|
||
curl -sf "${BASE}/question" 2>/dev/null
|
||
}
|
||
|
||
answer_question() {
|
||
local qid="$1"
|
||
local text="$2"
|
||
curl -sf -X POST "${BASE}/question/${qid}/answer" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"body\":\"${text}\"}" >/dev/null 2>&1
|
||
}
|
||
|
||
reject_question() {
|
||
local qid="$1"
|
||
curl -sf -X POST "${BASE}/question/${qid}/reject" >/dev/null 2>&1
|
||
}
|
||
|
||
# ── Forgejo configuration from environment + git remote fallback ─────────────
|
||
resolve_forgejo_config() {
|
||
local url="${FORGEJO_URL:-}"
|
||
local owner="${FORGEJO_OWNER:-}"
|
||
local repo="${FORGEJO_REPO:-}"
|
||
local pat="${FORGEJO_PAT:-}"
|
||
local git_name="${GIT_USER_NAME:-}"
|
||
local git_email="${GIT_USER_EMAIL:-}"
|
||
|
||
if [[ -z "$owner" || -z "$repo" ]]; then
|
||
local remote
|
||
remote=$(git remote get-url origin 2>/dev/null)
|
||
if [[ -n "$remote" ]]; then
|
||
local proto_host path_seg remote_owner remote_repo
|
||
proto_host=$(echo "$remote" | sed -n 's|^\(https\?://[^/]\+\).*|\1|p')
|
||
path_seg=$(echo "$remote" | sed -n 's|.*://[^/]\+/\([^/]\+/[^/]\+\).*|\1|p' | sed 's/\.git$//')
|
||
remote_owner="${path_seg%%/*}"
|
||
remote_repo="${path_seg#*/}"
|
||
url="${url:-${proto_host}}"
|
||
owner="${owner:-${remote_owner}}"
|
||
repo="${repo:-${remote_repo}}"
|
||
fi
|
||
fi
|
||
|
||
FORGEJO_URL="$url"
|
||
FORGEJO_OWNER="$owner"
|
||
FORGEJO_REPO="$repo"
|
||
FORGEJO_PAT="$pat"
|
||
GIT_USER_NAME="$git_name"
|
||
GIT_USER_EMAIL="$git_email"
|
||
}
|
||
|
||
# ── Work-group fetchers ────────────────────────────────────────────────────
|
||
fetch_list_script() {
|
||
local script_path="$1"
|
||
local forgejo_url="$2"
|
||
local forgejo_owner="$3"
|
||
local forgejo_repo="$4"
|
||
local forgejo_pat="$5"
|
||
|
||
if [[ -z "$forgejo_url" || -z "$forgejo_pat" ]]; then
|
||
echo "[]"
|
||
return
|
||
fi
|
||
|
||
local result
|
||
result=$(timeout "$SCRIPT_TIMEOUT_SEC" npx --yes tsx "$script_path" \
|
||
--url "$forgejo_url" \
|
||
--pat "$forgejo_pat" \
|
||
--owner "$forgejo_owner" \
|
||
--repo "$forgejo_repo" 2>/dev/null)
|
||
local status=$?
|
||
if [[ $status -eq 139 || $status -eq 137 || $status -eq 143 || $status -eq 124 ]]; then
|
||
log "[fetch_list_script] WARNING: $(basename "$script_path") timed out after ${SCRIPT_TIMEOUT_SEC}s (exit ${status})"
|
||
echo "[]"
|
||
return
|
||
fi
|
||
if [[ $status -ne 0 ]]; then
|
||
log "[fetch_list_script] WARNING: $(basename "$script_path") failed with exit ${status}"
|
||
echo "[]"
|
||
return
|
||
fi
|
||
local first_char="${result%"${result#?}"}"
|
||
if [[ "$first_char" == "[" ]]; then
|
||
echo "$result"
|
||
else
|
||
log "[fetch_list_script] WARNING: $(basename "$script_path") produced non-JSON output (starts with '${first_char}'); treating as empty"
|
||
echo "[]"
|
||
fi
|
||
}
|
||
|
||
# ── Worker prompt builders ─────────────────────────────────────────────────
|
||
# All fields (title, body) have newlines collapsed so the prompt file stays
|
||
# valid as a single-line-ish text file. The full item JSON is embedded compact
|
||
# (jq -c . → one line, no literal newlines) at the end of the prompt.
|
||
# session_start.ts reads this file as plain text and sends it to the agent.
|
||
build_impl_prompt() {
|
||
local item_json="$1"
|
||
local kind="$2"
|
||
local work_num="$3"
|
||
|
||
local work_type; work_type=$([ "$kind" == "ISSUE" ] && echo "issue_impl" || echo "pr_fix")
|
||
local title body milestone_id milestone_title labels_json compact_item
|
||
title=$(echo "$item_json" | jq -r '.title' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
|
||
body=$(echo "$item_json" | jq -r '.body // ""' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
|
||
milestone_id=$(echo "$item_json" | jq -r '.milestone.id // 0')
|
||
milestone_title=$(echo "$item_json" | jq -r '.milestone.title // ""' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
|
||
labels_json=$(echo "$item_json" | jq -c '.labels // []')
|
||
compact_item=$(echo "$item_json" | jq -c .)
|
||
|
||
cat <<PROMPT
|
||
issue_number: ${work_num}
|
||
pr_number: ${work_num}
|
||
work_type: ${work_type}
|
||
issue_title: ${title}
|
||
pr_title: ${title}
|
||
issue_body: ${body}
|
||
pr_body: ${body}
|
||
milestone_id: ${milestone_id}
|
||
milestone_title: ${milestone_title}
|
||
labels_json: ${labels_json}
|
||
forgejo_url: ${FORGEJO_URL}
|
||
forgejo_owner: ${FORGEJO_OWNER}
|
||
forgejo_repo: ${FORGEJO_REPO}
|
||
forgejo_pat: ${FORGEJO_PAT}
|
||
git_user_name: ${GIT_USER_NAME}
|
||
git_user_email: ${GIT_USER_EMAIL}
|
||
item_json: ${compact_item}
|
||
|
||
Implement or fix the indicated issue or pull request.
|
||
|
||
PR Compliance Checklist (MANDATORY — complete ALL items before creating a PR):
|
||
[ ] 1. CHANGELOG.md — add entry under [Unreleased] section
|
||
[ ] 2. CONTRIBUTORS.md — add or update contribution entry
|
||
[ ] 3. Commit footer — include \`ISSUES CLOSED: #<issue-number>\` in the commit message
|
||
[ ] 4. CI passes — all quality gates and tests green before requesting review
|
||
[ ] 5. BDD/Behave tests — added or updated for the changed behaviour
|
||
[ ] 6. Epic reference — PR description references the parent Epic issue number
|
||
[ ] 7. Labels — applied via forgejo-label-manager: State/In Review, Priority/<level>, MoSCoW/<level>, Type/<type>
|
||
[ ] 8. Milestone — PR assigned to the earliest open milestone matching the issue
|
||
PROMPT
|
||
}
|
||
|
||
build_merge_prompt() {
|
||
local pr_json="$1"
|
||
local pr_num="$2"
|
||
|
||
local title head_sha base_sha merge_base is_stale has_conflicts
|
||
title=$(echo "$pr_json" | jq -r '.title' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
|
||
head_sha=$(echo "$pr_json" | jq -r '.head.sha')
|
||
base_sha=$(echo "$pr_json" | jq -r '.base.sha')
|
||
merge_base=$(echo "$pr_json" | jq -r '.merge_base // empty')
|
||
[[ "$merge_base" == "empty" || -z "$merge_base" || "$merge_base" == "null" ]] && merge_base="$base_sha"
|
||
is_stale=$(echo "$pr_json" | jq -r 'if (.stale_state // "not_stale") == "not_stale" then "false" else "true" end')
|
||
has_conflicts=$(echo "$pr_json" | jq -r 'if .mergeable == false then "true" else "false" end')
|
||
|
||
local approvals_count ci_status review_status compact_item
|
||
approvals_count=$(echo "$pr_json" | jq -r '.approvals_count // 0')
|
||
ci_status=$(echo "$pr_json" | jq -r '.ci_status // "unknown"')
|
||
review_status=$(echo "$pr_json" | jq -r 'if (.approvals_count // 0) > 0 then "approved" else "not_approved" end')
|
||
compact_item=$(echo "$pr_json" | jq -c .)
|
||
|
||
cat <<PROMPT
|
||
pr_number: ${pr_num}
|
||
pr_title: ${title}
|
||
branch_name: ${head_sha}
|
||
head_sha: ${head_sha}
|
||
base_sha: ${base_sha}
|
||
merge_base_sha: ${merge_base}
|
||
is_stale: ${is_stale}
|
||
has_conflicts: ${has_conflicts}
|
||
review_status: ${review_status}
|
||
ci_status: ${ci_status}
|
||
approvals_count: ${approvals_count}
|
||
forgejo_url: ${FORGEJO_URL}
|
||
forgejo_owner: ${FORGEJO_OWNER}
|
||
forgejo_repo: ${FORGEJO_REPO}
|
||
forgejo_pat: ${FORGEJO_PAT}
|
||
git_user_name: ${GIT_USER_NAME}
|
||
git_user_email: ${GIT_USER_EMAIL}
|
||
item_json: ${compact_item}
|
||
|
||
Process the indicated Pull Request or Issue.
|
||
PROMPT
|
||
}
|
||
|
||
build_review_prompt() {
|
||
local pr_json="$1"
|
||
local pr_num="$2"
|
||
|
||
local title head_sha base_sha merge_base is_stale has_conflicts
|
||
title=$(echo "$pr_json" | jq -r '.title' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
|
||
head_sha=$(echo "$pr_json" | jq -r '.head.sha')
|
||
base_sha=$(echo "$pr_json" | jq -r '.base.sha')
|
||
merge_base=$(echo "$pr_json" | jq -r '.merge_base // empty')
|
||
[[ "$merge_base" == "empty" || -z "$merge_base" || "$merge_base" == "null" ]] && merge_base="$base_sha"
|
||
is_stale=$(echo "$pr_json" | jq -r 'if (.stale_state // "not_stale") == "not_stale" then "false" else "true" end')
|
||
has_conflicts=$(echo "$pr_json" | jq -r 'if .mergeable == false then "true" else "false" end')
|
||
|
||
local approvals_count ci_status compact_item
|
||
approvals_count=$(echo "$pr_json" | jq -r '.approvals_count // 0')
|
||
ci_status=$(echo "$pr_json" | jq -r '.ci_status // "unknown"')
|
||
compact_item=$(echo "$pr_json" | jq -c .)
|
||
|
||
cat <<PROMPT
|
||
pr_number: ${pr_num}
|
||
pr_title: ${title}
|
||
head_sha: ${head_sha}
|
||
base_sha: ${base_sha}
|
||
merge_base_sha: ${merge_base}
|
||
is_stale: ${is_stale}
|
||
has_conflicts: ${has_conflicts}
|
||
approvals_count: ${approvals_count}
|
||
ci_status: ${ci_status}
|
||
forgejo_url: ${FORGEJO_URL}
|
||
forgejo_owner: ${FORGEJO_OWNER}
|
||
forgejo_repo: ${FORGEJO_REPO}
|
||
forgejo_pat: ${FORGEJO_PAT}
|
||
git_user_name: ${GIT_USER_NAME}
|
||
git_user_email: ${GIT_USER_EMAIL}
|
||
item_json: ${compact_item}
|
||
|
||
Process the indicated Pull Request or Issue.
|
||
PROMPT
|
||
}
|
||
|
||
# ── Fleet analysis ────────────────────────────────────────────────────────
|
||
get_busy_worker_tags() {
|
||
local prefix="$1"
|
||
find_sessions_by_prefix "$prefix" "busy" | jq -r '.[].tag // empty'
|
||
}
|
||
|
||
extract_work_number() {
|
||
local tag="$1"
|
||
echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p'
|
||
}
|
||
|
||
# ── Duplicate worker guard ────────────────────────────────────────────────
|
||
deduplicate_workers() {
|
||
local prefix="$1"
|
||
local workers_json
|
||
workers_json=$(find_sessions_by_prefix "$prefix" "busy")
|
||
local duplicates
|
||
duplicates=$(echo "$workers_json" | jq -r '
|
||
group_by(.tag)
|
||
| map(select(length > 1))
|
||
| map(sort_by(.last_active) | reverse | .[1:])
|
||
| flatten
|
||
| .[].id
|
||
')
|
||
if [[ -n "$duplicates" ]]; then
|
||
while IFS= read -r dup_id; do
|
||
[[ -n "$dup_id" ]] || continue
|
||
log "Duplicate worker guard: deleting older session ${dup_id}"
|
||
delete_session "$dup_id"
|
||
done <<< "$duplicates"
|
||
fi
|
||
}
|
||
|
||
# ── Disk cleanup ──────────────────────────────────────────────────────────
|
||
cleanup_orphan_tmp_dirs() {
|
||
local prefix="$1"
|
||
local active_tags
|
||
active_tags=$(find_sessions_by_prefix "$prefix" "all" | jq -r '.[].tag // empty' | sort -u)
|
||
|
||
local tag_pattern="${prefix}"
|
||
while IFS= read -r -d '' dir; do
|
||
[[ -d "$dir" ]] || continue
|
||
local dir_id
|
||
dir_id=$(basename "$dir" | grep -oP '\d+' | tail -n1)
|
||
if [[ -n "$dir_id" ]]; then
|
||
local is_active=false
|
||
while IFS= read -r atag; do
|
||
[[ -n "$atag" ]] || continue
|
||
if [[ "$atag" == *"${dir_id}"* ]]; then
|
||
is_active=true
|
||
break
|
||
fi
|
||
done <<< "$active_tags"
|
||
if ! $is_active; then
|
||
log "Disk cleanup: removing orphan ${dir}"
|
||
rm -rf "$dir"
|
||
fi
|
||
fi
|
||
done < <(find /tmp -maxdepth 1 -type d -name "*${tag_pattern}*" -print0 2>/dev/null)
|
||
}
|
||
|
||
# ── Health checks ─────────────────────────────────────────────────────────
|
||
run_health_cycle() {
|
||
local now
|
||
now=$(date +%s)
|
||
local did_something=false
|
||
|
||
# 1) Question handling
|
||
local questions_json
|
||
questions_json=$(get_pending_questions)
|
||
if [[ -n "$questions_json" && "$questions_json" != "[]" && "$questions_json" != "null" ]]; then
|
||
local qids sids
|
||
qids=$(echo "$questions_json" | jq -r '.[].id')
|
||
sids=$(echo "$questions_json" | jq -r '.[].sessionID')
|
||
while IFS= read -r qid && IFS= read -r sid <&3; do
|
||
[[ -n "$qid" ]] || continue
|
||
[[ -n "$sid" ]] || continue
|
||
local cnt="${QUESTION_COUNT[$sid]:-0}"
|
||
if [[ "$cnt" -eq 0 ]]; then
|
||
log "Question from ${sid} (first time) — answering with anti-question text"
|
||
answer_question "$qid" "you are never under any circumstances to ask questions or pass along questions from subagents, use your best judgement and do not ask questions again"
|
||
QUESTION_COUNT[$sid]=1
|
||
else
|
||
log "Question from ${sid} (repeat) — deleting session"
|
||
delete_session "$sid"
|
||
unset QUESTION_COUNT[$sid]
|
||
fi
|
||
did_something=true
|
||
done <<< "$qids" 3<<< "$sids"
|
||
fi
|
||
|
||
# 2) Idle worker check — sessions with no recent activity
|
||
local all_prefixes=("AUTO-IMP" "AUTO-MRG" "AUTO-REV")
|
||
for prefix in "${all_prefixes[@]}"; do
|
||
local idle_workers
|
||
idle_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \
|
||
--arg cutoff "$(( (now * 1000) - IDLE_THRESHOLD_MS ))" \
|
||
'.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty')
|
||
while IFS= read -r sid; do
|
||
[[ -n "$sid" ]] || continue
|
||
local msgs
|
||
msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$sid" --limit 10 2>/dev/null)
|
||
if [[ -z "$msgs" || "$msgs" == "[]" ]]; then
|
||
log "Idle worker ${sid} has no messages — deleting"
|
||
delete_session "$sid"
|
||
did_something=true
|
||
continue
|
||
fi
|
||
local last_text
|
||
last_text=$(extract_last_assistant_text "$msgs")
|
||
if echo "$last_text" | grep -qiE '(done|finished|complete|exiting|all tests pass|committed|merged|review submitted)'; then
|
||
log "Idle worker ${sid} appears finished — deleting"
|
||
delete_session "$sid"
|
||
did_something=true
|
||
continue
|
||
fi
|
||
local now_s
|
||
now_s=$(date +%s)
|
||
local times_str="${CONTINUE_TIMES[$sid]:-}"
|
||
local cutoff=$((now_s - CONTINUE_BURST_WINDOW))
|
||
local new_times_arr=()
|
||
local count=0
|
||
if [[ -n "$times_str" ]]; then
|
||
for ts in $times_str; do
|
||
[[ -n "$ts" ]] || continue
|
||
if (( ts > cutoff )); then
|
||
new_times_arr+=("$ts")
|
||
((count++))
|
||
fi
|
||
done
|
||
fi
|
||
new_times_arr+=("$now_s")
|
||
((count++))
|
||
CONTINUE_TIMES[$sid]=$(printf '%s ' "${new_times_arr[@]}")
|
||
if [[ "$count" -gt $CONTINUE_BURST_LIMIT ]]; then
|
||
log "Idle worker ${sid} received ${count} continues in ${CONTINUE_BURST_WINDOW}s — deleting"
|
||
delete_session "$sid"
|
||
unset CONTINUE_TIMES[$sid]
|
||
did_something=true
|
||
else
|
||
log "Idle worker ${sid} — sending continue"
|
||
local cont_body
|
||
cont_body=$(jq -nc '{agent: "", parts: [{type: "text", text: "continue"}]}')
|
||
send_message_capture "$sid" "$cont_body" "/tmp/continue-${sid}.txt"
|
||
did_something=true
|
||
fi
|
||
done <<< "$idle_workers"
|
||
done
|
||
|
||
# 3) Doom-loop check (every 15 minutes)
|
||
if [[ $((now - LAST_DOOM_CHECK)) -ge $DOOM_CHECK_INTERVAL ]]; then
|
||
LAST_DOOM_CHECK=$now
|
||
for prefix in "${all_prefixes[@]}"; do
|
||
local busy_workers
|
||
busy_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \
|
||
--arg cutoff "$(( (now - DOOM_AGE_THRESHOLD) * 1000 ))" \
|
||
'.[] | select(.last_active >= ($cutoff | tonumber)) | .id // empty')
|
||
while IFS= read -r sid; do
|
||
[[ -n "$sid" ]] || continue
|
||
log "Doom-loop check for ${sid} (busy >30 min)"
|
||
local eval_tag="BUILDER-HEALTH-${sid: -6}"
|
||
local eval_title="[${eval_tag}] worker-health-evaluator"
|
||
local eval_prompt="session_id: ${sid}
|
||
|
||
Evaluate the health of this session."
|
||
local eval_session_json
|
||
eval_session_json=$(create_session "$eval_title")
|
||
local eval_sid
|
||
eval_sid=$(echo "$eval_session_json" | jq -r '.id')
|
||
if [[ -n "$eval_sid" && "$eval_sid" != "null" ]]; then
|
||
local eval_tmp="/tmp/builder-prompt-${eval_tag}-$$.txt"
|
||
printf '%s' "$eval_prompt" > "$eval_tmp"
|
||
local eval_body
|
||
eval_body=$(jq -nc --arg a "worker-health-evaluator" --rawfile t "$eval_tmp" '{agent: $a, parts: [{type: "text", text: $t}]}')
|
||
curl -sf -X POST "${BASE}/session/${eval_sid}/prompt_async" -H "Content-Type: application/json" -d "$eval_body" >/dev/null 2>&1
|
||
local eval_deadline=$(( $(date +%s) + 120 ))
|
||
while [[ $(date +%s) -lt $eval_deadline ]]; do
|
||
local eval_status
|
||
eval_status=$(curl -sf "${BASE}/session/status" 2>/dev/null | jq -r --arg sid "$eval_sid" '.[$sid].type // "idle"')
|
||
[[ "$eval_status" == "idle" ]] && break
|
||
sleep 2
|
||
done
|
||
local eval_msgs
|
||
eval_msgs=$($SESSION_MESSAGES --server "${BASE}" --session-id "$eval_sid" --limit 10 2>/dev/null)
|
||
local eval_text
|
||
eval_text=$(extract_last_assistant_text "$eval_msgs")
|
||
delete_session "$eval_sid"
|
||
rm -f "$eval_tmp"
|
||
local is_unrecoverable
|
||
is_unrecoverable=$(echo "$eval_text" | jq -r '.unrecoverable // false' 2>/dev/null || echo "false")
|
||
if [[ "$is_unrecoverable" == "true" ]]; then
|
||
log "Session ${sid} is unrecoverable — deleting"
|
||
delete_session "$sid"
|
||
did_something=true
|
||
fi
|
||
fi
|
||
done <<< "$busy_workers"
|
||
done
|
||
fi
|
||
|
||
if $did_something; then
|
||
return 0
|
||
else
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ── Preflight checks ─────────────────────────────────────────────────────
|
||
for cmd in curl jq opencode npx; do
|
||
command -v "$cmd" &>/dev/null || die "'$cmd' is required but not found in PATH"
|
||
done
|
||
|
||
# ── Start the opencode server ─────────────────────────────────────────────
|
||
if check_opencode_running; then
|
||
echo "┌─────────────────────────────────────────────────────────────────┐"
|
||
echo "│ ⚠ WARNING: OpenCode server already running on ${HOST}:${PORT}"
|
||
echo "│ 🔗 Connecting to existing server instead of starting new one"
|
||
echo "│ 💡 The script will NOT manage this server's lifecycle"
|
||
echo "└─────────────────────────────────────────────────────────────────┘"
|
||
echo ""
|
||
log "Using existing OpenCode server at ${HOST}:${PORT}"
|
||
OWN_SERVER=false
|
||
else
|
||
log "Starting opencode server on ${HOST}:${PORT}..."
|
||
opencode serve --port "$PORT" --hostname "$HOST" &
|
||
SERVER_PID=$!
|
||
OWN_SERVER=true
|
||
|
||
log "Waiting for server to become healthy (up to ${HEALTH_TIMEOUT}s)..."
|
||
healthy=false
|
||
for (( i = 1; i <= HEALTH_TIMEOUT; i++ )); do
|
||
if curl -sf "${BASE}/global/health" >/dev/null 2>&1; then
|
||
healthy=true
|
||
break
|
||
fi
|
||
kill -0 "$SERVER_PID" 2>/dev/null || die "Server process exited unexpectedly"
|
||
sleep 1
|
||
done
|
||
$healthy || die "Server did not become healthy within ${HEALTH_TIMEOUT}s"
|
||
log "Server is ready."
|
||
fi
|
||
|
||
# ── Main orchestration loop ───────────────────────────────────────────────
|
||
main_loop() {
|
||
log "Starting orchestration loop."
|
||
log "Worker pools: merge=${merge_max_workers} imp=${imp_max_workers} rev=${rev_max_workers}"
|
||
|
||
while ! $STOP_LOOP; do
|
||
N=$((N + 1))
|
||
|
||
idle_count=$(count_idle_events_last_minute)
|
||
if (( idle_count > MAX_IDLE_PER_MINUTE )); then
|
||
log "── iteration #${N} ── TOO MANY IDLE EVENTS (${idle_count}) — skipping cycle"
|
||
sleep 5
|
||
continue
|
||
fi
|
||
|
||
log "── iteration #${N} ──"
|
||
local did_anything=false
|
||
|
||
local all_sessions_json
|
||
all_sessions_json=$(query_all_sessions)
|
||
if [[ -z "$all_sessions_json" ]]; then
|
||
log "No sessions found."
|
||
fi
|
||
|
||
# ── Step 2: Per-type analysis & direct worker launch ──────────────────
|
||
log "[step2] resolving Forgejo config..."
|
||
resolve_forgejo_config
|
||
log "[step2] Forgejo config: url=${FORGEJO_URL} owner=${FORGEJO_OWNER} repo=${FORGEJO_REPO} pat_set=$([[ -n "$FORGEJO_PAT" ]] && echo yes || echo no)"
|
||
|
||
if [[ -z "$FORGEJO_PAT" ]]; then
|
||
log "FORGEJO_PAT not set — skipping worker launch"
|
||
else
|
||
declare -a IMP_WORK_GROUPS=(
|
||
"CI Failing PRs|${SCRIPT_DIR}/list_prs_ci_failing.ts|PR"
|
||
"Addressed Changes CI Passing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_passing.ts|PR"
|
||
"Addressed Changes CI Failing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_failing.ts|PR"
|
||
"Changes Requested|${SCRIPT_DIR}/list_prs_changes_requested.ts|PR"
|
||
"CI Missing|${SCRIPT_DIR}/list_prs_missing_ci_checks.ts|PR"
|
||
"No Active Review CI Passing|${SCRIPT_DIR}/list_prs_no_active_review_ci_passing.ts|PR"
|
||
"No Active Review CI Failing|${SCRIPT_DIR}/list_prs_no_active_review_ci_failing.ts|PR"
|
||
"Needs Review Not Stale|${SCRIPT_DIR}/list_prs_needs_review_not_stale.ts|PR"
|
||
"Needs Review Stale Clean|${SCRIPT_DIR}/list_prs_needs_review_stale_clean.ts|PR"
|
||
"Needs Review Stale Conflicts|${SCRIPT_DIR}/list_prs_needs_review_stale_conflicts.ts|PR"
|
||
"Open Issues|${SCRIPT_DIR}/list_issues.ts|ISSUE"
|
||
)
|
||
declare -a MRG_WORK_GROUPS=(
|
||
"Ready to Merge|${SCRIPT_DIR}/list_prs_ready_to_merge.ts|PR"
|
||
"Stale No Conflicts|${SCRIPT_DIR}/list_prs_stale_clean.ts|PR"
|
||
"Stale Has Conflicts|${SCRIPT_DIR}/list_prs_stale_conflicts.ts|PR"
|
||
)
|
||
declare -a REV_WORK_GROUPS=(
|
||
"No Active Review CI Passing|${SCRIPT_DIR}/list_prs_no_active_review_ci_passing.ts|PR"
|
||
"No Active Review CI Failing|${SCRIPT_DIR}/list_prs_no_active_review_ci_failing.ts|PR"
|
||
"Needs Review Not Stale|${SCRIPT_DIR}/list_prs_needs_review_not_stale.ts|PR"
|
||
"Needs Review Stale Clean|${SCRIPT_DIR}/list_prs_needs_review_stale_clean.ts|PR"
|
||
"Needs Review Stale Conflicts|${SCRIPT_DIR}/list_prs_needs_review_stale_conflicts.ts|PR"
|
||
"Addressed Changes CI Passing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_passing.ts|PR"
|
||
"Addressed Changes CI Failing|${SCRIPT_DIR}/list_prs_addressed_changes_ci_failing.ts|PR"
|
||
"Changes Requested|${SCRIPT_DIR}/list_prs_changes_requested.ts|PR"
|
||
"CI Failing|${SCRIPT_DIR}/list_prs_ci_failing.ts|PR"
|
||
)
|
||
|
||
declare -a pools=(
|
||
"AUTO-IMP|implementation-worker|${imp_max_workers}|IMP_WORK_GROUPS"
|
||
"AUTO-MRG|pr-merge-worker|${merge_max_workers}|MRG_WORK_GROUPS"
|
||
"AUTO-REV|pr-review-worker|${rev_max_workers}|REV_WORK_GROUPS"
|
||
)
|
||
|
||
for pool_spec in "${pools[@]}"; do
|
||
IFS='|' read -r prefix agent max wg_array_name <<< "$pool_spec"
|
||
|
||
log "[pool:${prefix}] starting pool processing (agent=${agent}, max=${max})"
|
||
log "[pool:${prefix}] deduplicate_workers..."
|
||
deduplicate_workers "$prefix"
|
||
log "[pool:${prefix}] cleanup_orphan_tmp_dirs..."
|
||
cleanup_orphan_tmp_dirs "$prefix"
|
||
|
||
local exclude_numbers=","
|
||
log "[pool:${prefix}] find_sessions_by_prefix '${prefix}' 'all'..."
|
||
local all_sessions_json
|
||
all_sessions_json=$(find_sessions_by_prefix "$prefix" "all")
|
||
local total_count
|
||
total_count=$(echo "$all_sessions_json" | jq 'length')
|
||
log "[pool:${prefix}] total sessions found: ${total_count}"
|
||
|
||
log "[pool:${prefix}] get_busy_worker_tags..."
|
||
local busy_tags
|
||
busy_tags=$(get_busy_worker_tags "$prefix")
|
||
local busy_count=0
|
||
while IFS= read -r tag; do
|
||
[[ -n "$tag" ]] || continue
|
||
local num
|
||
num=$(echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p')
|
||
if [[ -n "$num" ]]; then
|
||
exclude_numbers="${exclude_numbers}${num},"
|
||
((busy_count++))
|
||
fi
|
||
done <<< "$busy_tags"
|
||
log "[pool:${prefix}] busy_count=${busy_count} (from ${total_count} sessions)"
|
||
|
||
local available_slots=$((max - busy_count))
|
||
if [[ "$available_slots" -le 0 ]]; then
|
||
log "Pool ${prefix}: ${busy_count}/${max} active — full, skipping"
|
||
continue
|
||
fi
|
||
|
||
log "Pool ${prefix}: ${busy_count}/${max} active (${total_count} total sessions, ${available_slots} slots free)"
|
||
did_anything=true
|
||
|
||
local wg_array_ref="${wg_array_name}[@]"
|
||
local work_groups=("${!wg_array_ref}")
|
||
|
||
# ── PRE-FETCH: launch ALL list scripts in parallel, cache results ────
|
||
# Each list_prs_*.ts script fetches ALL open PRs from Forgejo (~90s) and
|
||
# applies a filter. Running them sequentially = 11×90s = 990s.
|
||
# Running them in parallel = ~90s total (one round of Forgejo API calls).
|
||
log "[pool:${prefix}] pre-fetching ${#work_groups[@]} work-group scripts in parallel..."
|
||
local -A script_cache
|
||
local -a cache_pids
|
||
local -a cache_keys
|
||
local cache_dir="/tmp/opencode-builder-cache-$$"
|
||
mkdir -p "$cache_dir"
|
||
|
||
for type_spec in "${work_groups[@]}"; do
|
||
IFS='|' read -r wg_label wg_scripts wg_kind <<< "$type_spec"
|
||
local scripts_arr
|
||
IFS=',' read -ra scripts_arr <<< "$wg_scripts"
|
||
for script_path in "${scripts_arr[@]}"; do
|
||
[[ -n "$script_path" ]] || continue
|
||
local cache_key="${script_path##*/}" # just filename
|
||
cache_key="${cache_key%.ts}"
|
||
local out_file="${cache_dir}/${cache_key}.json"
|
||
cache_keys+=("$cache_key")
|
||
log "[pool:${prefix}] launching $(basename "$script_path") (cache key: ${cache_key})..."
|
||
timeout "$SCRIPT_TIMEOUT_SEC" npx --yes tsx "$script_path" \
|
||
--url "$FORGEJO_URL" \
|
||
--pat "$FORGEJO_PAT" \
|
||
--owner "$FORGEJO_OWNER" \
|
||
--repo "$FORGEJO_REPO" \
|
||
> "$out_file" 2>/dev/null &
|
||
cache_pids+=($!)
|
||
done
|
||
done
|
||
|
||
log "[pool:${prefix}] waiting for ${#cache_pids[@]} pre-fetch jobs to complete..."
|
||
local cache_idx=0
|
||
for pid in "${cache_pids[@]}"; do
|
||
wait "$pid" 2>/dev/null
|
||
local ck="${cache_keys[$cache_idx]}"
|
||
local out_file="${cache_dir}/${ck}.json"
|
||
if [[ -s "$out_file" ]]; then
|
||
local first_char
|
||
first_char=$(head -c1 "$out_file" 2>/dev/null)
|
||
if [[ "$first_char" == "[" ]]; then
|
||
log "[pool:${prefix}] ${ck}: ready ($(wc -l < "$out_file") lines)"
|
||
else
|
||
log "[pool:${prefix}] ${ck}: invalid output (starts with '${first_char}'), treating as empty"
|
||
echo "[]" > "$out_file"
|
||
fi
|
||
else
|
||
log "[pool:${prefix}] ${ck}: empty or timeout, treating as empty"
|
||
echo "[]" > "$out_file"
|
||
fi
|
||
((cache_idx++))
|
||
done
|
||
log "[pool:${prefix}] pre-fetch complete, starting worker selection..."
|
||
|
||
# ── Worker selection: iterate cached work-groups ─────────────────────
|
||
local launched=0
|
||
for type_spec in "${work_groups[@]}"; do
|
||
if (( launched >= available_slots )); then
|
||
break
|
||
fi
|
||
|
||
IFS='|' read -r wg_label wg_scripts wg_kind <<< "$type_spec"
|
||
log "[pool:${prefix}] work-group '${wg_label}': selecting workers..."
|
||
|
||
local scripts_arr
|
||
IFS=',' read -ra scripts_arr <<< "$wg_scripts"
|
||
for script_path in "${scripts_arr[@]}"; do
|
||
if (( launched >= available_slots )); then
|
||
break 2
|
||
fi
|
||
[[ -n "$script_path" ]] || continue
|
||
|
||
local cache_key="${script_path##*/}"
|
||
cache_key="${cache_key%.ts}"
|
||
local out_file="${cache_dir}/${cache_key}.json"
|
||
local json
|
||
json=$(cat "$out_file" 2>/dev/null || echo "[]")
|
||
|
||
local raw_count
|
||
raw_count=$(echo "$json" | jq 'length' 2>/dev/null || echo 0)
|
||
log "[pool:${prefix}] script $(basename "$script_path"): ${raw_count} items in cache"
|
||
|
||
if [[ "$raw_count" -eq 0 || "$raw_count" == "null" || "$raw_count" == "" ]]; then
|
||
continue
|
||
fi
|
||
|
||
local -a items_to_launch=()
|
||
while IFS= read -r item; do
|
||
[[ -n "$item" ]] || continue
|
||
|
||
local work_num
|
||
work_num=$(echo "$item" | jq -r '.number')
|
||
if [[ -z "$work_num" || "$work_num" == "null" ]]; then
|
||
continue
|
||
fi
|
||
if echo ",${exclude_numbers}," | grep -q ",${work_num},"; then
|
||
continue
|
||
fi
|
||
|
||
local kind="$wg_kind"
|
||
if [[ "$kind" == "AUTO" ]]; then
|
||
if echo "$item" | jq -e '.head // empty' >/dev/null 2>&1; then
|
||
kind="PR"
|
||
else
|
||
kind="ISSUE"
|
||
fi
|
||
fi
|
||
|
||
items_to_launch+=("${item}|${work_num}|${kind}")
|
||
done < <(echo "$json" | jq -c '.[]')
|
||
|
||
if [[ ${#items_to_launch[@]} -eq 0 ]]; then
|
||
continue
|
||
fi
|
||
|
||
local batch_count
|
||
if [[ ${#items_to_launch[@]} -lt $((available_slots - launched)) ]]; then
|
||
batch_count=${#items_to_launch[@]}
|
||
else
|
||
batch_count=$((available_slots - launched))
|
||
fi
|
||
|
||
log "[pool:${prefix}] '${wg_label}': launching ${batch_count} workers in parallel (${#items_to_launch[@]} candidates)"
|
||
|
||
local -a pids=()
|
||
local -a tmp_files=()
|
||
local -a work_nums_arr=()
|
||
|
||
for ((i = 0; i < batch_count; i++)); do
|
||
local item_with_meta="${items_to_launch[i]}"
|
||
local item="${item_with_meta%%|*}"
|
||
local rest="${item_with_meta#*|}"
|
||
local work_num="${rest%%|*}"
|
||
local kind="${rest##*|}"
|
||
|
||
local worker_tag="${prefix}-${kind}-${work_num}"
|
||
local tmp_prompt="/tmp/worker-prompt-${worker_tag// /-}-$$-${i}.txt"
|
||
|
||
local prompt_text
|
||
case "$agent" in
|
||
*implementation*) prompt_text=$(build_impl_prompt "$item" "$kind" "$work_num") ;;
|
||
*merge*) prompt_text=$(build_merge_prompt "$item" "$work_num") ;;
|
||
*review*) prompt_text=$(build_review_prompt "$item" "$work_num") ;;
|
||
esac
|
||
|
||
printf '%s' "$prompt_text" > "$tmp_prompt"
|
||
|
||
$SESSION_START --server "${BASE}" --tag "$worker_tag" --agent "$agent" --prompt-file "$tmp_prompt" --restart \
|
||
>/dev/null 2>&1 &
|
||
pids+=($!)
|
||
tmp_files+=("$tmp_prompt")
|
||
work_nums_arr+=("$work_num")
|
||
done
|
||
|
||
for ((i = 0; i < ${#pids[@]}; i++)); do
|
||
if wait "${pids[i]}" 2>/dev/null; then
|
||
exclude_numbers="${exclude_numbers}${work_nums_arr[i]},"
|
||
((launched++))
|
||
did_anything=true
|
||
log "[${prefix}] launched ${work_nums_arr[i]} (${agent})"
|
||
else
|
||
log "[${prefix}] FAILED ${work_nums_arr[i]}"
|
||
fi
|
||
rm -f "${tmp_files[i]}"
|
||
done
|
||
done
|
||
done
|
||
|
||
log "Pool ${prefix}: done — ${launched} workers launched this iteration"
|
||
rm -rf "$cache_dir"
|
||
done
|
||
fi
|
||
|
||
# ── Step 3: Health & maintenance cycle ────────────────────────────────
|
||
if run_health_cycle; then
|
||
did_anything=true
|
||
fi
|
||
|
||
# ── Step 4: Idle sleep if nothing happened ────────────────────────────
|
||
if ! $did_anything; then
|
||
log "Nothing to do — sleeping ${SLEEP_WHEN_IDLE}s"
|
||
record_idle_event
|
||
sleep "$SLEEP_WHEN_IDLE"
|
||
fi
|
||
done
|
||
|
||
log "Loop stopped. Shutting down."
|
||
exit 0
|
||
}
|
||
|
||
main_loop |