fix: parallel pre-fetch, timeouts, newline-safe prompts, verbose logging

opencode-builder.sh: major reliability and correctness fixes:

- Parallel pre-fetch per pool: ALL list_prs_*.ts scripts for a pool now
  launch in parallel (background jobs), results cached to temp files. Each
  pool iteration fetches all work-group data in ONE parallel batch (~90s)
  instead of 11 sequential ~90s calls (~990s). The bottleneck is Forgejo
  API speed, not sequential script execution.

- Timeouts everywhere: SCRIPT_TIMEOUT_SEC raised 60s→300s (Forgejo is slow).
  find_sessions_by_prefix and other session_* scripts get 30s timeout via
  the timeout(1) utility. fetch_list_script catches timeout exit codes
  (124, 137, 139, 143) and returns [] gracefully.

- Prompt builders fixed: build_impl_prompt, build_merge_prompt,
  build_review_prompt now collapse newlines in title/body via tr+sed so
  the prompt file stays valid as plain text. Full item JSON embedded as
  compact single-line (jq -c .) at end of prompt.

- Verbose step logging: [step2], [pool:NAME] prefixes throughout main
  loop show exactly which stage is running and where any hang occurs.

- BUILDER_DEBUG=1 env var enables bash -x tracing for deep debugging.

Note: find_sessions_by_prefix --status busy returns empty (OpenCode
/session/status only reports the supervisor session). busy_count=0 when
all session status fields are empty, making all slots available. The
total session count is logged alongside active count for visibility.
This commit is contained in:
2026-05-12 23:18:16 +00:00
committed by CleverThis
parent ebce3ae4a7
commit b86f067bfb
+160 -76
View File
@@ -9,12 +9,20 @@
set -uo pipefail
# ── Debug mode ─────────────────────────────────────────────────────────────
if [[ "${BUILDER_DEBUG:-0}" == "1" ]]; then
set -x
log "DEBUG MODE ENABLED"
fi
# ── Configuration (override via environment) ──────────────────────────────
PORT="${OPENCODE_PORT:-4096}"
HOST="${OPENCODE_HOST:-127.0.0.1}"
BASE="http://${HOST}:${PORT}"
HEALTH_TIMEOUT=60
MAX_IDLE_PER_MINUTE=10
SCRIPT_TIMEOUT_SEC=300 # timeout for list_prs / list_issues scripts (Forgejo is slow — allow 5 min)
SESSION_SCRIPT_TIMEOUT=30 # timeout for session_* scripts
# Worker pool sizing
total_max_workers="${CA_MAX_PARALLEL_WORKERS:-4}"
@@ -129,7 +137,7 @@ query_all_sessions() {
find_sessions_by_prefix() {
local prefix="$1"
local status_filter="${2:-all}"
$SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null
timeout "$SESSION_SCRIPT_TIMEOUT" $SESSION_FIND_PREFIX --server "${BASE}" --prefix "$prefix" --exclude-supervisor --status "$status_filter" 2>/dev/null
}
get_busy_worker_tags() {
@@ -278,54 +286,68 @@ fetch_list_script() {
fi
local result
result=$(npx --yes tsx "$script_path" \
result=$(timeout "$SCRIPT_TIMEOUT_SEC" npx --yes tsx "$script_path" \
--url "$forgejo_url" \
--pat "$forgejo_pat" \
--owner "$forgejo_owner" \
--repo "$forgejo_repo" 2>/dev/null)
local status=$?
if [[ $status -eq 139 || $status -eq 137 || $status -eq 143 || $status -eq 124 ]]; then
log "[fetch_list_script] WARNING: $(basename "$script_path") timed out after ${SCRIPT_TIMEOUT_SEC}s (exit ${status})"
echo "[]"
return
fi
if [[ $status -ne 0 ]]; then
log "[fetch_list_script] WARNING: $(basename "$script_path") failed with exit ${status}"
echo "[]"
return
fi
local first_char="${result%"${result#?}"}"
if [[ "$first_char" == "[" ]]; then
echo "$result"
else
log "[fetch_list_script] WARNING: $(basename "$script_path") produced non-JSON output (starts with '${first_char}'); treating as empty"
echo "[]"
fi
}
# ── Worker prompt builders ─────────────────────────────────────────────────
# All fields (title, body) have newlines collapsed so the prompt file stays
# valid as a single-line-ish text file. The full item JSON is embedded compact
# (jq -c . → one line, no literal newlines) at the end of the prompt.
# session_start.ts reads this file as plain text and sends it to the agent.
build_impl_prompt() {
local item_json="$1"
local kind="$2"
local work_num="$3"
local title body milestone milestone_id milestone_title labels_json
title=$(echo "$item_json" | jq -r '.title')
body=$(echo "$item_json" | jq -r '.body // ""')
if [[ "$kind" == "ISSUE" ]]; then
milestone=$(echo "$item_json" | jq -r '.milestone')
else
milestone=$(echo "$item_json" | jq -r '.milestone // null')
fi
milestone_id=$(echo "$milestone" | jq -r 'if . == null then 0 else .id end')
milestone_title=$(echo "$milestone" | jq -r 'if . == null then "" else .title end')
labels_json=$(echo "$item_json" | jq -r '.labels')
local work_type; work_type=$([ "$kind" == "ISSUE" ] && echo "issue_impl" || echo "pr_fix")
local title body milestone_id milestone_title labels_json compact_item
title=$(echo "$item_json" | jq -r '.title' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
body=$(echo "$item_json" | jq -r '.body // ""' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
milestone_id=$(echo "$item_json" | jq -r '.milestone.id // 0')
milestone_title=$(echo "$item_json" | jq -r '.milestone.title // ""' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
labels_json=$(echo "$item_json" | jq -c '.labels // []')
compact_item=$(echo "$item_json" | jq -c .)
cat <<PROMPT
issue_number: ${work_num}
pr_number: ${work_num}
work_type: $([ "$kind" == "ISSUE" ] && echo "issue_impl" || echo "pr_fix")
issue_title: "${title//\"/\\\"}"
pr_title: "${title//\"/\\\"}"
issue_body: "${body//\"/\\\"}"
pr_body: "${body//\"/\\\"}"
work_type: ${work_type}
issue_title: ${title}
pr_title: ${title}
issue_body: ${body}
pr_body: ${body}
milestone_id: ${milestone_id}
milestone_title: "${milestone_title}"
milestone_title: ${milestone_title}
labels_json: ${labels_json}
forgejo_url: "${FORGEJO_URL}"
forgejo_owner: "${FORGEJO_OWNER}"
forgejo_repo: "${FORGEJO_REPO}"
forgejo_pat: "${FORGEJO_PAT}"
git_user_name: "${GIT_USER_NAME}"
git_user_email: "${GIT_USER_EMAIL}"
forgejo_url: ${FORGEJO_URL}
forgejo_owner: ${FORGEJO_OWNER}
forgejo_repo: ${FORGEJO_REPO}
forgejo_pat: ${FORGEJO_PAT}
git_user_name: ${GIT_USER_NAME}
git_user_email: ${GIT_USER_EMAIL}
item_json: ${compact_item}
Implement or fix the indicated issue or pull request.
@@ -345,43 +367,40 @@ build_merge_prompt() {
local pr_json="$1"
local pr_num="$2"
local title branch_name head_sha base_sha merge_base is_stale has_conflicts
title=$(echo "$pr_json" | jq -r '.title')
branch_name=$(echo "$pr_json" | jq -r '.head.ref')
local title head_sha base_sha merge_base is_stale has_conflicts
title=$(echo "$pr_json" | jq -r '.title' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
head_sha=$(echo "$pr_json" | jq -r '.head.sha')
base_sha=$(echo "$pr_json" | jq -r '.base.sha')
merge_base=$(echo "$pr_json" | jq -r '.merge_base // empty')
[[ "$merge_base" == "empty" || -z "$merge_base" || "$merge_base" == "null" ]] && merge_base="$base_sha"
is_stale=$(echo "$pr_json" | jq -r 'if (.stale_state // "not_stale") == "not_stale" then "false" else "true" end')
has_conflicts=$(echo "$pr_json" | jq -r 'if .mergeable == false then "true" else "false" end')
local stale_state mergeable
stale_state=$(echo "$pr_json" | jq -r '.stale_state // "not_stale"')
mergeable=$(echo "$pr_json" | jq -r '.mergeable')
[[ "$stale_state" == "not_stale" ]] && is_stale="false" || is_stale="true"
[[ "$mergeable" == "false" ]] && has_conflicts="true" || has_conflicts="false"
local approvals_count ci_status review_status
local approvals_count ci_status review_status compact_item
approvals_count=$(echo "$pr_json" | jq -r '.approvals_count // 0')
ci_status=$(echo "$pr_json" | jq -r '.ci_status // "unknown"')
[[ "$approvals_count" -gt 0 ]] && review_status="approved" || review_status="not_approved"
review_status=$(echo "$pr_json" | jq -r 'if (.approvals_count // 0) > 0 then "approved" else "not_approved" end')
compact_item=$(echo "$pr_json" | jq -c .)
cat <<PROMPT
pr_number: ${pr_num}
pr_title: "${title//\"/\\\"}"
branch_name: "${branch_name}"
head_sha: "${head_sha}"
base_sha: "${base_sha}"
merge_base_sha: "${merge_base}"
pr_title: ${title}
branch_name: ${head_sha}
head_sha: ${head_sha}
base_sha: ${base_sha}
merge_base_sha: ${merge_base}
is_stale: ${is_stale}
has_conflicts: ${has_conflicts}
review_status: "${review_status}"
ci_status: "${ci_status}"
review_status: ${review_status}
ci_status: ${ci_status}
approvals_count: ${approvals_count}
forgejo_url: "${FORGEJO_URL}"
forgejo_owner: "${FORGEJO_OWNER}"
forgejo_repo: "${FORGEJO_REPO}"
forgejo_pat: "${FORGEJO_PAT}"
git_user_name: "${GIT_USER_NAME}"
git_user_email: "${GIT_USER_EMAIL}"
forgejo_url: ${FORGEJO_URL}
forgejo_owner: ${FORGEJO_OWNER}
forgejo_repo: ${FORGEJO_REPO}
forgejo_pat: ${FORGEJO_PAT}
git_user_name: ${GIT_USER_NAME}
git_user_email: ${GIT_USER_EMAIL}
item_json: ${compact_item}
Process the indicated Pull Request or Issue.
PROMPT
@@ -391,41 +410,37 @@ build_review_prompt() {
local pr_json="$1"
local pr_num="$2"
local title branch_name head_sha base_sha merge_base is_stale has_conflicts
title=$(echo "$pr_json" | jq -r '.title')
branch_name=$(echo "$pr_json" | jq -r '.head.ref')
local title head_sha base_sha merge_base is_stale has_conflicts
title=$(echo "$pr_json" | jq -r '.title' | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g')
head_sha=$(echo "$pr_json" | jq -r '.head.sha')
base_sha=$(echo "$pr_json" | jq -r '.base.sha')
merge_base=$(echo "$pr_json" | jq -r '.merge_base // empty')
[[ "$merge_base" == "empty" || -z "$merge_base" || "$merge_base" == "null" ]] && merge_base="$base_sha"
is_stale=$(echo "$pr_json" | jq -r 'if (.stale_state // "not_stale") == "not_stale" then "false" else "true" end')
has_conflicts=$(echo "$pr_json" | jq -r 'if .mergeable == false then "true" else "false" end')
local stale_state mergeable
stale_state=$(echo "$pr_json" | jq -r '.stale_state // "not_stale"')
mergeable=$(echo "$pr_json" | jq -r '.mergeable')
[[ "$stale_state" == "not_stale" ]] && is_stale="false" || is_stale="true"
[[ "$mergeable" == "false" ]] && has_conflicts="true" || has_conflicts="false"
local approvals_count ci_status
local approvals_count ci_status compact_item
approvals_count=$(echo "$pr_json" | jq -r '.approvals_count // 0')
ci_status=$(echo "$pr_json" | jq -r '.ci_status // "unknown"')
compact_item=$(echo "$pr_json" | jq -c .)
cat <<PROMPT
pr_number: ${pr_num}
pr_title: "${title//\"/\\\"}"
branch_name: "${branch_name}"
head_sha: "${head_sha}"
base_sha: "${base_sha}"
merge_base_sha: "${merge_base}"
pr_title: ${title}
head_sha: ${head_sha}
base_sha: ${base_sha}
merge_base_sha: ${merge_base}
is_stale: ${is_stale}
has_conflicts: ${has_conflicts}
approvals_count: ${approvals_count}
ci_status: "${ci_status}"
forgejo_url: "${FORGEJO_URL}"
forgejo_owner: "${FORGEJO_OWNER}"
forgejo_repo: "${FORGEJO_REPO}"
forgejo_pat: "${FORGEJO_PAT}"
git_user_name: "${GIT_USER_NAME}"
git_user_email: "${GIT_USER_EMAIL}"
ci_status: ${ci_status}
forgejo_url: ${FORGEJO_URL}
forgejo_owner: ${FORGEJO_OWNER}
forgejo_repo: ${FORGEJO_REPO}
forgejo_pat: ${FORGEJO_PAT}
git_user_name: ${GIT_USER_NAME}
git_user_email: ${GIT_USER_EMAIL}
item_json: ${compact_item}
Process the indicated Pull Request or Issue.
PROMPT
@@ -698,7 +713,9 @@ main_loop() {
fi
# ── Step 2: Per-type analysis & direct worker launch ──────────────────
log "[step2] resolving Forgejo config..."
resolve_forgejo_config
log "[step2] Forgejo config: url=${FORGEJO_URL} owner=${FORGEJO_OWNER} repo=${FORGEJO_REPO} pat_set=$([[ -n "$FORGEJO_PAT" ]] && echo yes || echo no)"
if [[ -z "$FORGEJO_PAT" ]]; then
log "FORGEJO_PAT not set — skipping worker launch"
@@ -742,15 +759,21 @@ main_loop() {
for pool_spec in "${pools[@]}"; do
IFS='|' read -r prefix agent max wg_array_name <<< "$pool_spec"
log "[pool:${prefix}] starting pool processing (agent=${agent}, max=${max})"
log "[pool:${prefix}] deduplicate_workers..."
deduplicate_workers "$prefix"
log "[pool:${prefix}] cleanup_orphan_tmp_dirs..."
cleanup_orphan_tmp_dirs "$prefix"
local exclude_numbers=","
log "[pool:${prefix}] find_sessions_by_prefix '${prefix}' 'all'..."
local all_sessions_json
all_sessions_json=$(find_sessions_by_prefix "$prefix" "all")
local total_count
total_count=$(echo "$all_sessions_json" | jq 'length')
log "[pool:${prefix}] total sessions found: ${total_count}"
log "[pool:${prefix}] get_busy_worker_tags..."
local busy_tags
busy_tags=$(get_busy_worker_tags "$prefix")
local busy_count=0
@@ -763,6 +786,7 @@ main_loop() {
((busy_count++))
fi
done <<< "$busy_tags"
log "[pool:${prefix}] busy_count=${busy_count} (from ${total_count} sessions)"
local available_slots=$((max - busy_count))
if [[ "$available_slots" -le 0 ]]; then
@@ -776,6 +800,62 @@ main_loop() {
local wg_array_ref="${wg_array_name}[@]"
local work_groups=("${!wg_array_ref}")
# ── PRE-FETCH: launch ALL list scripts in parallel, cache results ────
# Each list_prs_*.ts script fetches ALL open PRs from Forgejo (~90s) and
# applies a filter. Running them sequentially = 11×90s = 990s.
# Running them in parallel = ~90s total (one round of Forgejo API calls).
log "[pool:${prefix}] pre-fetching ${#work_groups[@]} work-group scripts in parallel..."
local -A script_cache
local -a cache_pids
local -a cache_keys
local cache_dir="/tmp/opencode-builder-cache-$$"
mkdir -p "$cache_dir"
for type_spec in "${work_groups[@]}"; do
IFS='|' read -r wg_label wg_scripts wg_kind <<< "$type_spec"
local scripts_arr
IFS=',' read -ra scripts_arr <<< "$wg_scripts"
for script_path in "${scripts_arr[@]}"; do
[[ -n "$script_path" ]] || continue
local cache_key="${script_path##*/}" # just filename
cache_key="${cache_key%.ts}"
local out_file="${cache_dir}/${cache_key}.json"
cache_keys+=("$cache_key")
log "[pool:${prefix}] launching $(basename "$script_path") (cache key: ${cache_key})..."
timeout "$SCRIPT_TIMEOUT_SEC" npx --yes tsx "$script_path" \
--url "$FORGEJO_URL" \
--pat "$FORGEJO_PAT" \
--owner "$FORGEJO_OWNER" \
--repo "$FORGEJO_REPO" \
> "$out_file" 2>/dev/null &
cache_pids+=($!)
done
done
log "[pool:${prefix}] waiting for ${#cache_pids[@]} pre-fetch jobs to complete..."
local cache_idx=0
for pid in "${cache_pids[@]}"; do
wait "$pid" 2>/dev/null
local ck="${cache_keys[$cache_idx]}"
local out_file="${cache_dir}/${ck}.json"
if [[ -s "$out_file" ]]; then
local first_char
first_char=$(head -c1 "$out_file" 2>/dev/null)
if [[ "$first_char" == "[" ]]; then
log "[pool:${prefix}] ${ck}: ready ($(wc -l < "$out_file") lines)"
else
log "[pool:${prefix}] ${ck}: invalid output (starts with '${first_char}'), treating as empty"
echo "[]" > "$out_file"
fi
else
log "[pool:${prefix}] ${ck}: empty or timeout, treating as empty"
echo "[]" > "$out_file"
fi
((cache_idx++))
done
log "[pool:${prefix}] pre-fetch complete, starting worker selection..."
# ── Worker selection: iterate cached work-groups ─────────────────────
local launched=0
for type_spec in "${work_groups[@]}"; do
if (( launched >= available_slots )); then
@@ -783,7 +863,7 @@ main_loop() {
fi
IFS='|' read -r wg_label wg_scripts wg_kind <<< "$type_spec"
log "[${prefix}] checking work-group: '${wg_label}'"
log "[pool:${prefix}] work-group '${wg_label}': selecting workers..."
local scripts_arr
IFS=',' read -ra scripts_arr <<< "$wg_scripts"
@@ -793,14 +873,17 @@ main_loop() {
fi
[[ -n "$script_path" ]] || continue
local cache_key="${script_path##*/}"
cache_key="${cache_key%.ts}"
local out_file="${cache_dir}/${cache_key}.json"
local json
json=$(fetch_list_script "$script_path" "$FORGEJO_URL" "$FORGEJO_OWNER" "$FORGEJO_REPO" "$FORGEJO_PAT")
json=$(cat "$out_file" 2>/dev/null || echo "[]")
local raw_count
raw_count=$(echo "$json" | jq 'length' 2>/dev/null || echo 0)
log "[${prefix}] script $(basename "$script_path"): ${raw_count} items fetched"
log "[pool:${prefix}] script $(basename "$script_path"): ${raw_count} items in cache"
if [[ "$raw_count" -eq 0 || "$raw_count" == "null" ]]; then
if [[ "$raw_count" -eq 0 || "$raw_count" == "null" || "$raw_count" == "" ]]; then
continue
fi
@@ -840,7 +923,7 @@ main_loop() {
batch_count=$((available_slots - launched))
fi
log "[${prefix}] '${wg_label}': launching ${batch_count} workers in parallel (${#items_to_launch[@]} items in group)"
log "[pool:${prefix}] '${wg_label}': launching ${batch_count} workers in parallel (${#items_to_launch[@]} candidates)"
local -a pids=()
local -a tmp_files=()
@@ -887,6 +970,7 @@ main_loop() {
done
log "Pool ${prefix}: done — ${launched} workers launched this iteration"
rm -rf "$cache_dir"
done
fi