fix: count workers by last_active timestamp, not status API
The OpenCode /session/status endpoint reports all async workers as "idle" even while they are actively processing — making the previous --status busy filter always return empty and workers count as 0. Fixes across the board: - get_active_worker_tags() and count_active_workers(): new functions that filter by last_active >= (now - 2 min) instead of status field. - Pool launcher: use count_active_workers() for the busy count, and extract exclude_numbers from active sessions' tags. - Health cycle idle check: was using --status idle (returns all sessions since everything is "idle"). Now uses --status all + last_active threshold to find genuinely idle workers. - Doom loop check: was using --status busy (returns nothing). Now uses --status all + last_active < (now - 30 min) to find abandoned sessions. - fetch_list_script: guard against corrupted JSON when stderr messages from the script contaminate stdout — only accept output starting with "[" as valid JSON. - Worker launch: add --restart so stale sessions are always replaced with fresh workers. - Add IDLE_THRESHOLD_MS (3 min) constant for the health cycle.
This commit is contained in:
+59
-15
@@ -34,6 +34,7 @@ DOOM_AGE_THRESHOLD=1800 # 30 minutes
|
||||
CONTINUE_BURST_WINDOW=60 # seconds
|
||||
CONTINUE_BURST_LIMIT=5
|
||||
SLEEP_WHEN_IDLE=30 # seconds
|
||||
IDLE_THRESHOLD_MS=180000 # 3 minutes — session considered idle if last_active > this
|
||||
|
||||
# Script paths
|
||||
SCRIPT_DIR="/app/.opencode/skills/auto-agents-system/scripts"
|
||||
@@ -138,6 +139,28 @@ find_sessions_by_prefix() {
|
||||
$SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null
|
||||
}
|
||||
|
||||
ACTIVE_THRESHOLD_SECONDS=120
|
||||
|
||||
get_active_worker_tags() {
|
||||
local prefix="$1"
|
||||
local now_ms
|
||||
now_ms=$(date +%s000)
|
||||
local cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000))
|
||||
find_sessions_by_prefix "$prefix" "all" | jq -r \
|
||||
--arg cutoff "$cutoff_ms" \
|
||||
'.[] | select(.last_active > ($cutoff_ms | tonumber)) | .tag // empty'
|
||||
}
|
||||
|
||||
count_active_workers() {
|
||||
local prefix="$1"
|
||||
local now_ms
|
||||
now_ms=$(date +%s000)
|
||||
local cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000))
|
||||
find_sessions_by_prefix "$prefix" "all" | jq \
|
||||
--arg cutoff "$cutoff_ms" \
|
||||
'[.[] | select(.last_active > ($cutoff_ms | tonumber))] | length'
|
||||
}
|
||||
|
||||
create_session() {
|
||||
local title="$1"
|
||||
curl -sf -X POST "${BASE}/session" \
|
||||
@@ -287,11 +310,19 @@ fetch_list_script() {
|
||||
return
|
||||
fi
|
||||
|
||||
npx --yes tsx "$script_path" \
|
||||
local result
|
||||
result=$(npx --yes tsx "$script_path" \
|
||||
--url "$forgejo_url" \
|
||||
--pat "$forgejo_pat" \
|
||||
--owner "$forgejo_owner" \
|
||||
--repo "$forgejo_repo" 2>/dev/null
|
||||
--repo "$forgejo_repo" 2>/dev/null)
|
||||
local first_char
|
||||
first_char="${result%"${result#?}"}"
|
||||
if [[ "$first_char" == "[" || "$first_char" == "" ]]; then
|
||||
echo "$result"
|
||||
else
|
||||
echo "[]"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Worker launchers ───────────────────────────────────────────────────────
|
||||
@@ -570,11 +601,13 @@ run_health_cycle() {
|
||||
done <<< "$qids" 3<<< "$sids"
|
||||
fi
|
||||
|
||||
# 2) Idle worker check
|
||||
# 2) Idle worker check — use last_active threshold to find genuinely idle sessions
|
||||
local all_prefixes=("AUTO-IMP" "AUTO-MRG" "AUTO-REV")
|
||||
for prefix in "${all_prefixes[@]}"; do
|
||||
local idle_workers
|
||||
idle_workers=$(find_sessions_by_prefix "$prefix" "idle" | jq -r '.[].id // empty')
|
||||
idle_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \
|
||||
--arg cutoff "$(($(date +%s) * 1000 - IDLE_THRESHOLD_MS))" \
|
||||
'.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty')
|
||||
while IFS= read -r sid; do
|
||||
[[ -n "$sid" ]] || continue
|
||||
local msgs
|
||||
@@ -630,12 +663,14 @@ run_health_cycle() {
|
||||
done <<< "$idle_workers"
|
||||
done
|
||||
|
||||
# 3) Doom-loop check (every 15 minutes)
|
||||
# 3) Doom-loop check (every 15 minutes) — active sessions with no recent tool calls
|
||||
if [[ $((now - LAST_DOOM_CHECK)) -ge $DOOM_CHECK_INTERVAL ]]; then
|
||||
LAST_DOOM_CHECK=$now
|
||||
for prefix in "${all_prefixes[@]}"; do
|
||||
local busy_workers
|
||||
busy_workers=$(find_sessions_by_prefix "$prefix" "busy" | jq -r '.[] | select(.last_active < ('"$now"' * 1000 - '"$((DOOM_AGE_THRESHOLD * 1000))"')) | .id // empty')
|
||||
busy_workers=$(find_sessions_by_prefix "$prefix" "all" | jq -r \
|
||||
--arg cutoff "$(( (now - DOOM_AGE_THRESHOLD) * 1000 ))" \
|
||||
'.[] | select(.last_active < ($cutoff | tonumber)) | .id // empty')
|
||||
while IFS= read -r sid; do
|
||||
[[ -n "$sid" ]] || continue
|
||||
log "Doom-loop check for ${sid} (busy >30 min)"
|
||||
@@ -799,18 +834,27 @@ main_loop() {
|
||||
deduplicate_workers "$prefix"
|
||||
cleanup_orphan_tmp_dirs "$prefix"
|
||||
|
||||
local busy_tags exclude_numbers=","
|
||||
busy_tags=$(get_busy_worker_tags "$prefix")
|
||||
local busy_count=0
|
||||
local exclude_numbers=","
|
||||
local all_tags_json
|
||||
all_tags_json=$(find_sessions_by_prefix "$prefix" "all")
|
||||
local now_ms cutoff_ms
|
||||
now_ms=$(date +%s000)
|
||||
cutoff_ms=$((now_ms - ACTIVE_THRESHOLD_SECONDS * 1000))
|
||||
local active_tags_json
|
||||
active_tags_json=$(echo "$all_tags_json" | jq --arg cutoff "$cutoff_ms" \
|
||||
'[.[] | select(.last_active > ($cutoff_ms | tonumber))]')
|
||||
local busy_count
|
||||
busy_count=$(echo "$active_tags_json" | jq 'length')
|
||||
while IFS= read -r tag; do
|
||||
[[ -n "$tag" ]] || continue
|
||||
local num
|
||||
num=$(extract_work_number "$tag")
|
||||
num=$(echo "$tag" | sed -n 's/.*-\(PR\|ISSUE\)-\([0-9]\+\)$/\2/p')
|
||||
if [[ -n "$num" ]]; then
|
||||
exclude_numbers="${exclude_numbers}${num},"
|
||||
((busy_count++))
|
||||
fi
|
||||
done <<< "$busy_tags"
|
||||
done < <(echo "$active_tags_json" | jq -r '.[].tag // empty')
|
||||
|
||||
log "Pool ${prefix}: ${busy_count} active (${max} max) — excluded: $(echo "$active_tags_json" | jq 'length') recent"
|
||||
|
||||
local available_slots=$((max - busy_count))
|
||||
if [[ "$available_slots" -le 0 ]]; then
|
||||
@@ -906,7 +950,7 @@ main_loop() {
|
||||
|
||||
printf '%s' "$prompt_text" > "$tmp_prompt"
|
||||
|
||||
$SESSION_START --server "${BASE}" --tag "$worker_tag" --agent "$agent" --prompt-file "$tmp_prompt" \
|
||||
$SESSION_START --server "${BASE}" --tag "$worker_tag" --agent "$agent" --prompt-file "$tmp_prompt" --restart \
|
||||
>/dev/null 2>&1 &
|
||||
pids+=($!)
|
||||
tmp_files+=("$tmp_prompt")
|
||||
@@ -918,9 +962,9 @@ main_loop() {
|
||||
exclude_numbers="${exclude_numbers}${work_nums[i]},"
|
||||
((launched++))
|
||||
did_anything=true
|
||||
log "Worker [${prefix}-${work_nums[i]}] launched successfully"
|
||||
log "[${prefix}] launched worker ${work_nums[i]} (${agent})"
|
||||
else
|
||||
log "Worker [${prefix}-${work_nums[i]}] failed"
|
||||
log "[${prefix}] FAILED worker ${work_nums[i]}"
|
||||
fi
|
||||
rm -f "${tmp_files[i]}"
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user