From 3d2f9ae043ef71420f2e2cab67f4d2fedf32dc98 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Tue, 12 May 2026 19:27:56 +0000 Subject: [PATCH] fix: count workers by last_active timestamp, not status API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenCode /session/status endpoint reports all async workers as "idle" even while they are actively processing — making the previous --status busy filter always return empty and workers count as 0. Fixes across the board: - get_active_worker_tags() and count_active_workers(): new functions that filter by last_active >= (now - 2 min) instead of status field. - Pool launcher: use count_active_workers() for the busy count, and extract exclude_numbers from active sessions' tags. - Health cycle idle check: was using --status idle (returns all sessions since everything is "idle"). Now uses --status all + last_active threshold to find genuinely idle workers. - Doom loop check: was using --status busy (returns nothing). Now uses --status all + last_active < (now - 30 min) to find abandoned sessions. - fetch_list_script: guard against corrupted JSON when stderr messages from the script contaminate stdout — only accept output starting with "[" as valid JSON. - Worker launch: add --restart so stale sessions are always replaced with fresh workers. - Add IDLE_THRESHOLD_MS (3 min) constant for the health cycle. --- scripts/opencode-builder.sh | 74 +++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/scripts/opencode-builder.sh b/scripts/opencode-builder.sh index ddc3eba49..2ad3e51ce 100755 --- a/scripts/opencode-builder.sh +++ b/scripts/opencode-builder.sh @@ -34,6 +34,7 @@ DOOM_AGE_THRESHOLD=1800 # 30 minutes CONTINUE_BURST_WINDOW=60 # seconds CONTINUE_BURST_LIMIT=5 SLEEP_WHEN_IDLE=30 # seconds +IDLE_THRESHOLD_MS=180000 # 3 minutes — session considered idle if last_active > this # Script paths SCRIPT_DIR="/app/.opencode/skills/auto-agents-system/scripts" @@ -138,6 +139,28 @@ find_sessions_by_prefix() { $SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null } +ACTIVE_THRESHOLD_SECONDS=120 + +get_active_worker_tags() { + local prefix="$1" + local now_ms + now_ms=$(date +%s000) + local cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000)) + find_sessions_by_prefix "$prefix" "all" | jq -r \ + --arg cutoff "$cutoff_ms" \ + '.[] | select(.last_active > ($cutoff_ms | tonumber)) | .tag // empty' +} + +count_active_workers() { + local prefix="$1" + local now_ms + now_ms=$(date +%s000) + local cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000)) + find_sessions_by_prefix "$prefix" "all" | jq \ + --arg cutoff "$cutoff_ms" \ + '[.[] | select(.last_active > ($cutoff_ms | tonumber))] | length' +} + create_session() { local title="$1" curl -sf -X POST "${BASE}/session" \ @@ -287,11 +310,19 @@ fetch_list_script() { return fi - npx --yes tsx "$script_path" \ + local result + result=$(npx --yes tsx "$script_path" \ --url "$forgejo_url" \ --pat "$forgejo_pat" \ --owner "$forgejo_owner" \ - --repo "$forgejo_repo" 2>/dev/null + --repo "$forgejo_repo" 2>/dev/null) + local first_char + first_char="${result%"${result#?}"}" + if [[ "$first_char" == "[" || "$first_char" == "" ]]; then + echo "$result" + else + echo "[]" + fi } # ── Worker launchers ─────────────────────────────────────────────────────── @@ -570,11 +601,13 @@ run_health_cycle() { done <<< "$qids" 3<<< "$sids" fi - # 2) Idle worker check + # 2) Idle worker check — use last_active threshold to find genuinely idle sessions local all_prefixes=("AUTO-IMP" "AUTO-MRG" "AUTO-REV") for prefix in "${all_prefixes[@]}"; do local idle_workers - idle_workers=$(find_sessions_by_prefix "$prefix" "idle" | jq -r '.[].id // empty') + idle_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \ + --arg cutoff "$(($(date +%s) * 1000 - IDLE_THRESHOLD_MS))" \ + '.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty') while IFS= read -r sid; do [[ -n "$sid" ]] || continue local msgs @@ -630,12 +663,14 @@ run_health_cycle() { done <<< "$idle_workers" done - # 3) Doom-loop check (every 15 minutes) + # 3) Doom-loop check (every 15 minutes) — active sessions with no recent tool calls if [[ $((now - LAST_DOOM_CHECK)) -ge $DOOM_CHECK_INTERVAL ]]; then LAST_DOOM_CHECK=$now for prefix in "${all_prefixes[@]}"; do local busy_workers - busy_workers=$(find_sessions_by_prefix "$prefix" "busy" | jq -r '.[] | select(.last_active < ('"$now"' * 1000 - '"$((DOOM_AGE_THRESHOLD * 1000))"')) | .id // empty') + busy_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \ + --arg cutoff "$(( (now - DOOM_AGE_THRESHOLD) * 1000 ))" \ + '.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty') while IFS= read -r sid; do [[ -n "$sid" ]] || continue log "Doom-loop check for ${sid} (busy >30 min)" @@ -799,18 +834,27 @@ main_loop() { deduplicate_workers "$prefix" cleanup_orphan_tmp_dirs "$prefix" - local busy_tags exclude_numbers="," - busy_tags=$(get_busy_worker_tags "$prefix") - local busy_count=0 + local exclude_numbers="," + local all_tags_json + all_tags_json=$(find_sessions_by_prefix "$prefix" "all") + local now_ms cutoff_ms + now_ms=$(date +%s000) + cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000)) + local active_tags_json + active_tags_json=$(echo "$all_tags_json" | jq --arg cutoff "$cutoff_ms" \ + '[.[] | select(.last_active > ($cutoff_ms | tonumber))]') + local busy_count + busy_count=$(echo "$active_tags_json" | jq 'length') while IFS= read -r tag; do [[ -n "$tag" ]] || continue local num - num=$(extract_work_number "$tag") + num=$(echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p') if [[ -n "$num" ]]; then exclude_numbers="${exclude_numbers}${num}," - ((busy_count++)) fi - done <<< "$busy_tags" + done < <(echo "$active_tags_json" | jq -r '.[].tag // empty') + + log "Pool ${prefix}: ${busy_count} active (${max} max) — excluded: $(echo "$active_tags_json" | jq 'length') recent" local available_slots=$((max - busy_count)) if [[ "$available_slots" -le 0 ]]; then @@ -906,7 +950,7 @@ main_loop() { printf '%s' "$prompt_text" > "$tmp_prompt" - $SESSION_START --server "${BASE}" --tag "$worker_tag" --agent "$agent" --prompt-file "$tmp_prompt" \ + $SESSION_START --server "${BASE}" --tag "$worker_tag" --agent "$agent" --prompt-file "$tmp_prompt" --restart \ >/dev/null 2>&1 & pids+=($!) tmp_files+=("$tmp_prompt") @@ -918,9 +962,9 @@ main_loop() { exclude_numbers="${exclude_numbers}${work_nums[i]}," ((launched++)) did_anything=true - log "Worker [${prefix}-${work_nums[i]}] launched successfully" + log "[${prefix}] launched worker ${work_nums[i]} (${agent})" else - log "Worker [${prefix}-${work_nums[i]}] failed" + log "[${prefix}] FAILED worker ${work_nums[i]}" fi rm -f "${tmp_files[i]}" done